#
# $QNXLicenseA:
# Copyright 2007, QNX Software Systems. All Rights Reserved.
# 
# You must obtain a written license from and pay applicable license fees to QNX 
# Software Systems before you may reproduce, modify or distribute this software, 
# or any work that includes all or part of this software.   Free development 
# licenses are available for evaluation and non-commercial purposes.  For more 
# information visit http://licensing.qnx.com or email licensing@qnx.com.
#  
# This file may contain contributions from others.  Please review this entire 
# file for other proprietary rights or license notices, as well as the QNX 
# Development Suite License Guide at http://licensing.qnx.com/license-guide/ 
# for other information.
# $
#

#
# Kernel exception entry/exit code.
#
# General exceptions are handled by __exc_general_start.
# This code is copied to VBR+SH_EXC_GENERAL by install_traps().
#
# TLB miss exceptions are handled by code in vm4.s
# This code is copied to VBR+SH_EXC_TLBMISS by init_vm().
#
# Interrupt handling is performed by code dynamically generated by gen_intr():
# - exception entry code from intr_entry_start
# - interrupt identification generated from the startup interrupt callouts
# - interrupt dispatch code in intr_process_queue and interrupt()
# - return from interrupt code in intr_done
#
# Exception entry uses bank1 registers (SR.BL=1, SR.RB=1):
#	r0_bank1 - available for use as temp register
#	r1_bank1 - available for use as temp register
#	r2_bank1 - available for use as temp register
#	r3_bank1 - available for use as temp register
#	r4_bank1 - &actives[RUNCPU]
#	r5_bank1 - ker_stack[RUNCPU]
#	r6_bank1 - RUNCPU << 2
#	r7_bank1 - &inkernel (SMP), inkernal value (non-SMP)
#
# Once in bank0, register usage is typically as follows:
#	r11 - pointer to saved register context
#	r12 - pointer to actives[RUNCPU]
#	r13 - old inkernel (exceptions), kernel/user entry (interrupts)
#
#
# Incomplete code is marked as follows:
# FIXME_SMP: issues affecting general SMP implementation
# FIXME_MSGOPT: issues related to SMP_MSGOPT implementation
#
	.global intr_entry_start
	.global intr_entry_end
	.global	__exc_general_start
	.global	__exc_general_end
	.global __exc_unexpected
	.global restore_fpu_registers

	.extern ker_stack
	.extern actives
	.extern actives_prp
	.extern aspaces_prp
	.extern cpupageptr
	.extern	memmgr
	.extern resched
	.extern kererr
	.extern fpusave_alloc
	.extern usr_fault
	.extern intrevent_drain
	.extern specialret
	.extern kdebug_callout
	.extern xfer_handlers
	.extern debug_attach_brkpts
	.extern debug_detach_brkpts
	.extern fpuemul
	.extern	interrupt
	.extern	ker_call_table
	.extern	emulate_instruction
	.extern	emulate_alignment
	.extern cpu_mutex_adjust
	.extern __shadow_imask
	.extern ker_savefpu

.ifdef VARIANT_instr
	.extern _trace_call_table
	.extern ker_exit_enable_mask
	.extern _trace_ker_exit
.endif

.ifdef VARIANT_smp
	.extern	ker_stack_top
	.extern	ker_stack_top
	.extern	run_ker_stack_bot
	.extern	run_ker_stack_bot
.endif

	.include "asmoff.def"
	.include "sh/util.ah"

.ifdef VARIANT_smp
	#
	# Data definitions for cpunum and inkernel storage
	# byte 0 - contains interrupt nesting count and inkernel bits
	# byte 1 - currently unused
	# byte 2 - currently unused
	# byte 3 - contains cpu number of cpu currently in the kernel
	#
	.globl	inkernel
	.type	inkernel, @object
	.size	inkernel,4
	.globl	cpunum
	.type	cpunum, @object
	.size	cpunum,1

.data
inkernel:	.byte 0
			.byte 0
			.byte 0
cpunum:		.byte 0

	#
	# Set reg to RUNCPU << 2
	#
.macro	SMP_GETCPU reg
	stc		r6_bank, &reg
.endm

	#
	# Set reg to RUNCPU << 2 when executing in bank1
	#
.macro	SMP_GETCPU_BANK1	reg
	mov		r6, &reg
.endm

	#
	# Get address indexed by cpu
	# The 'cpu' register must already be set by a previous SMP_GETCPU
	#
.macro	SMP_ADD		dst, cpu
	add		&cpu, &dst
.endm

#
# NOTE on movli.l/movco.l and SMP
#
# When we use movli/movco to implement a spin lock, we need to be careful to
# avoid a deadlock on SMP systems.  On uniprocessors, the movli sets the
# LDST flag.  This flag gets cleared on an interrupt or exception.  The movco
# only proceeds with the store if the LDST flag is set, so we can be sure
# we haven't been interrupted between the movli and movco.  On SMP, the
# LDST flag is also cleared if there is cache activity on another core on
# the same cache line that the movli instruction used.  It doesn't have to
# be the same address, just the same cache line.  This can lead to a deadlock
# between two cores trying to use movli/movco to modify different variables
# that just happen to share a cache line.
#
# To avoid this problem, we try to do all of our spinning before we execute
# the movli instruction.  We read the spinlock and test to see if we might
# try to lock it.  If it looks like we can, we then go into the normal
# spinlock lock code and use movli/movco to try to lock it.  If this fails
# (another CPU might be trying the exact same thing at the same time) we
# jump right back to the beginning to start over.
#

.macro	ACQUIRE_INTR_SLOCK	label
	#
	# This is a recursive spin lock - see kercpu.h
	#
	mov.l	&label, r1
	SMP_GETCPU	r2			! RUNCPU << 2
	shlr2	r2				! r2 = RUNCPU
	# First, test to see if we think we can grab the lock:
	# check to see if it's free or if we already own it.
0:	mov.l	@r1, r0
	tst		r0, r0
	bt		2f				! lock is free
	mov		r0, r3
	shlr16	r3
	shlr8	r3
	cmp/eq	r3, r2
	bf		0b				! lock is owned by another CPU

	# Looks good to try to grab the lock.
2:	.word	0x0163			! movli.l @r1, r0
	tst		r0, r0
	bt		1f				! lock is free
	mov		r0, r3
	shlr16	r3
	shlr8	r3				! extract cpuid (lock >> 24)
	cmp/eq	r3, r2
	bf		0b				! another cpu owns lock

	#
	# Lock is free or we already own it
	#
1:	add		#1, r0			! increment counter
	mov		r2, r3
	shll16	r3
	shll8	r3
	or		r3, r0			! make sure cpuid is set
	.word	0x0173			! movco.l r0, @r1
	bf		0b
.endm

.macro	RELEASE_INTR_SLOCK	label
	#
	# This is a recursive spin lock - see kercpu.h
	#
	mov.l	&label, r1
	mov.l	@r1, r0
	add		#-1, r0			! decrement counter
	mov		r0, r2
	shll8	r2				! counter value << 8
	tst		r2, r2
	bf		0f
	mov		#0, r0			! clear lock
0:	mov.l	r0, @r1
.endm

.else
	#
	# Uniprocessor versions of SMP-related macros
	#
.macro	SMP_GETCPU	reg
.endm
.macro	SMP_GETCPU_BANK1	reg
.endm
.macro	SMP_ADD		dst, cpu
.endm
.macro	ACQUIRE_INTR_SLOCK	label
.endm
.macro	RELEASE_INTR_SLOCK	label
.endm

.endif

#----------------------------------------------------------------------------
# Macro to save register context to area pointed by r1
#
# We are executing in bank1 (SR.BL=1, SR.RB=1)
#
# On return:
#	r1 - pointer to base of saved context
#----------------------------------------------------------------------------

.macro SAVECONTEXT
	sts.l	pr, @-r1
	sts.l	macl, @-r1
	sts.l	mach, @-r1
	stc.l	gbr, @-r1
	stc.l	spc, @-r1
	stc.l	ssr, @-r1
	mov.l	r15, @-r1
	mov.l	r14, @-r1
	mov.l	r13, @-r1
	mov.l	r12, @-r1
	mov.l	r11, @-r1
	mov.l	r10, @-r1
	mov.l	r9, @-r1
	mov.l	r8, @-r1
	stc.l	r7_bank, @-r1
	stc.l	r6_bank, @-r1
	stc.l	r5_bank, @-r1
	stc.l	r4_bank, @-r1
	stc.l	r3_bank, @-r1
	stc.l	r2_bank, @-r1
	stc.l	r1_bank, @-r1
	stc.l	r0_bank, @-r1
.endm

#----------------------------------------------------------------------------
# Macro for saving context for general exception entry
#
# We are executing in bank1 (SR.BL=1, SR.RB=1)
#
# On entry:
#	r3  - EXPEVT code (if we entered via VBR exception vector)
#
# On return:
#	Executing in bank1 (SR.BL=1, SR.RB=1)
#	r11 - pointer to saved context
#	r12 - actives[RUNCPU]
#	r14 - indicates mode on entry: 1 - kernel, 0 - user
#----------------------------------------------------------------------------

.macro EXC_SAVECONTEXT
.ifdef	VARIANT_smp
	#
	# We can't check inkernel on SMP since another cpu may have the kernel.
	# Instead we check if ker_stack_bot < r15 <= ker_stack_top
	#
	mov.l	102f, r0					! &ker_stack_bot
	mov.l	103f, r1					! &ker_stack_top
	mov.l	@r0, r0
	mov.l	@r1, r1
	cmp/hs	r0, r15						! is r15 > ker_stack_bot?
	bf		0f
	cmp/hi	r1, r15						! is r15 > ker_stack_top?
	bt		0f
.else
	tst		r7, r7
	bt		0f
.endif

	#
	# Already on kernel stack
	#
	mov		r15, r1
	SAVECONTEXT
	mov		r1, r15
	mov		r1, r11						! r11 = saved context
	mov.l	@r4, r12					! r12 = actives[RUNCPU]
	bra		1f
	mov		#1, r14						! indicate entry from kernel

	#
	# Save context in act->reg
	#
0:	mov.l	@r4, r2						! actives[RUNCPU]
	mov.l	101f, r0
	mov		r2, r1
	add		r0, r1						! &act->reg + SIZEOF_REG
	SAVECONTEXT
	mov		r5, r15						! ker_stack[RUNCPU]
	mov		r1, r11						! r11 = saved context
	mov		r2, r12						! r12 = actives[RUNCPU]
	mov		#0, r14						! indicate entry from user

	#
	# Check if cmpxchg emulation was active
	#
	mov.l	100f, r8				! __ker_emu_save
	SMP_GETCPU_BANK1	r0
	SMP_ADD	r8, r0
	mov.l	@r8, r8					! __ker_emu_save[RUNCPU]
	mov.l	@r8, r10				! active indicator
	tst		r10, r10
	bt		1f

	#
	# Restore context from __ker_emu_save[RUNCPU]
	#
	mov		#0, r10
	mov.l	r10, @r8				! clear active indicator
	add		#4, r8
	mov.l	@r8+, r10				! saved pc
	mov		r11, r1
	add		#REG_SR, r1
	mov.l	r10, @(4, r1)
	mov.l	@r8+, r10				! saved sr
	mov.l	r10, @(0, r1)
	mov.l	@r8+, r10				! saved r1
	mov.l	r10, @(REG_GR+4, r11)
	mov.l	@r8, r10				! saved r0
	mov.l	r10, @(REG_GR, r11)
	bra		1f
	nop

	.align 2

100:	.long	__ker_emu_save
101:	.long	REG_OFF+SIZEOF_REG
.ifdef	VARIANT_smp
102:	.long	ker_stack_bot
103:	.long	ker_stack_top
.endif

1:
.endm

#----------------------------------------------------------------------------
# Macro for acquiring kernel for general exception entry
#
# We are executing in bank1 (SR.BL=1, SR.RB=1)
#
# On return:
#   Executing in bank0 with interrupts disabled
#	r11 - pointer to saved context
#	r12 - actives[RUNCPU]
#	r13 - original inkernel value
#	r14 - indicates mode on entry: 1 - kernel, 0 - user
#----------------------------------------------------------------------------

.macro EXC_ENTERKERNEL
.ifdef	VARIANT_smp
	#
	# Call appropriate code to acquire kernel based on mode on entry
	# On return, we are in bank0 with interrupts disabled
	#
	tst		r14, r14
	bt		0f

	#
	# Entered from kernel/interrupt
	#
	mov.l	104f, r1
	mov		#0, r0
	jsr		@r1
	or		#INKERNEL_NOW|INKERNEL_LOCK, r0
	bra		2f
	mov		#1, r14

	#
	# Entered from user
	#
0:	mov.l	105f, r1
	mov		#0, r0
	jsr		@r1
	or		#INKERNEL_NOW|INKERNEL_LOCK, r0
	bra		2f
	mov		#0, r14
.else
	#
	# Acquire the kernel and switch to bank0 with interrupts disabled
	#
	mov		r7, r13
	mov		r13, r0
	or		#INKERNEL_NOW|INKERNEL_LOCK, r0
	mov		r0, r7

	#
	# Switch to bank0 with interrupts disabled
	#
	mov.l	106f, r1
	mov.l	107f, r2
	stc		sr, r0
	and		r1, r0					! turn off BL/RB bit
	or		r2, r0					! disable interrupts and FPU
	ldc		r0, sr

	bra		2f
	nop
.endif

	.align 2
.ifdef	VARIANT_smp
104:	.long	sys_acquire_kernel
105:	.long	usr_acquire_kernel
.else
106:	.long	~(SH_SR_BL | SH_SR_RB)
107:	.long	(SH_SR_FD | SH_SR_IMASK)
.endif
2:
.endm


#
# FPU contexts
#
# We handle thread and kernel FPU contexts in different manners, though they are
# similar.  In general we do not save an FPU context until a different context
# needs to use the FPU.
#
# In entering a kernel context (kernel call, exception or interrupt), we disable 
# the FPU (set the SR.FD bit).   If the kernel call or interrupt tries to use the 
# FPU we get an FPU-disabled exception and save the previous context at that point.
#
# The FPU-disabled exception code needs to find the appropriate place to save
# the previous FPU context.  When we enable the FPU for a thread, we set the
# actives_fpu variable to point to the thread entry object.  When we interrupt
# a kernel context, we set aside space on the stack to save the FPU context if
# necessary, and point the ker_savefpu variable at this space.
#
# On taking an FPU-disabled exception, if actives_fpu is non-null, we store the 
# FPU context to actives_fpu->fpudata (and then set actives_fpu=null).  If the 
# FPU-disabled exception came from a kernel context (i.e. kernel call, exception 
# or interrupt) and ker_savefpu is non-null, and the context hasn't already been 
# saved, we store the FPU context to ker_savefpu.  It should not be possible for 
# actives_fpu and ker_savefpu to be non-null at the same time.
#
# When a thread is made running during __ker_exit, we disable the FPU for that
# thread.  Thus when it tries to use the FPU again it takes another FPU-disabled
# exception.  This allows us to restore the FPU context from thp->fpudata.
#
# The ker_savefpu (per-CPU) variable is used to track the location to save the 
# FPU context required by a kernel call or interrupt.  We currently do not handle 
# the use of the FPU by an exception context.
#
# When an interrupt occurs we disable the FPU so any use of the FPU by an ISR
# will trigger an fpu-disabled exception.  If we were in a kernel context (kernel 
# call, exception or interrupt), and that context had the FPU enabled, we set aside 
# space on the stack to save the FPU context and assign ker_savefpu to point 
# to this save area.  This is done in the interrupt() routine in 
# ker/sh/interrupt.c.  If an fpu-disabled exception subsequently occurs, we will
# store the FPU context in this area (through ker_savefpu).  Before the interrupt()
# routine returns, if the context-save area was used, we restore the FPU context
# just before the routine returns.
# 


.EQU	HALF_SH_FPU_REG_SIZE, SH_FPU_REG_SIZE/2

# macro to save the FPU registers to the context area pointed to by the
# register given as a parameter.  Note: also trashes r0
.macro SAVE_FPU_REGISTERS	reg
	# have to add the size of the register save area in two halves, since
	# it's too big for an immediate operand
	add		#HALF_SH_FPU_REG_SIZE, &reg
	add		#HALF_SH_FPU_REG_SIZE, &reg
	sts.l	fpscr, @-&reg
	sts.l	fpul, @-&reg

	#
	# Save bank 1
	#
	mov.l	100f, r0
	lds		r0, fpscr
	fmov	fr15, @-&reg
	fmov	fr14, @-&reg
	fmov	fr13, @-&reg
	fmov	fr12, @-&reg
	fmov	fr11, @-&reg
	fmov	fr10, @-&reg
	fmov	fr9, @-&reg
	fmov	fr8, @-&reg
	fmov	fr7, @-&reg
	fmov	fr6, @-&reg
	fmov	fr5, @-&reg
	fmov	fr4, @-&reg
	fmov	fr3, @-&reg
	fmov	fr2, @-&reg
	fmov	fr1, @-&reg
	fmov	fr0, @-&reg

	#
	# Save bank 0
	#
	mov.l	101f, r0
	lds		r0, fpscr
	fmov	fr15, @-&reg
	fmov	fr14, @-&reg
	fmov	fr13, @-&reg
	fmov	fr12, @-&reg
	fmov	fr11, @-&reg
	fmov	fr10, @-&reg
	fmov	fr9, @-&reg
	fmov	fr8, @-&reg
	fmov	fr7, @-&reg
	fmov	fr6, @-&reg
	fmov	fr5, @-&reg
	fmov	fr4, @-&reg
	fmov	fr3, @-&reg
	fmov	fr2, @-&reg
	fmov	fr1, @-&reg
	fmov	fr0, @-&reg
	
	bra		109f
	nop

	.align 2
100:	.long	SH_FPSCR_DN | SH_FPSCR_PR | SH_FPSCR_FR 
101:	.long	SH_FPSCR_DN | SH_FPSCR_PR 
	
109:

.endm


# macro to restore the FPU registers from a context pointed to by
# the register given as a parameter.  Note: also trashes r0
.macro RESTORE_FPU_REGISTERS	reg
	# restore bank 0
	mov.l	100f,r0
	lds		r0,fpscr
	fmov	@&reg+,fr0
	fmov	@&reg+,fr1
	fmov	@&reg+,fr2
	fmov	@&reg+,fr3
	fmov	@&reg+,fr4
	fmov	@&reg+,fr5
	fmov	@&reg+,fr6
	fmov	@&reg+,fr7
	fmov	@&reg+,fr8
	fmov	@&reg+,fr9
	fmov	@&reg+,fr10
	fmov	@&reg+,fr11
	fmov	@&reg+,fr12
	fmov	@&reg+,fr13
	fmov	@&reg+,fr14
	fmov	@&reg+,fr15
	# restore bank 1
	mov.l	101f,r0
	lds		r0,fpscr
	fmov	@&reg+,fr0
	fmov	@&reg+,fr1
	fmov	@&reg+,fr2
	fmov	@&reg+,fr3
	fmov	@&reg+,fr4
	fmov	@&reg+,fr5
	fmov	@&reg+,fr6
	fmov	@&reg+,fr7
	fmov	@&reg+,fr8
	fmov	@&reg+,fr9
	fmov	@&reg+,fr10
	fmov	@&reg+,fr11
	fmov	@&reg+,fr12
	fmov	@&reg+,fr13
	fmov	@&reg+,fr14
	fmov	@&reg+,fr15
	# restore fpul and fpscr 
	lds.l	@&reg+,fpul
	lds.l	@&reg+,fpscr
	bra 109f
	nop
	
	.align 2
100:	.long	SH_FPSCR_DN | SH_FPSCR_PR 					! bank 0 fpscr value
101:	.long	SH_FPSCR_DN | SH_FPSCR_PR | SH_FPSCR_FR 	! bank 1 fpscr value
	
109:
.endm


.text

#-------------------------------------------------------------------------
#
# ker_start - begin execution of initial thread for each cpu
#
# On entry:
#
# r7_bank1 = inkernel (non-SMP), &inkernel (SMP)
#
# This routine sets up the following:
#
# r4_bank1 = &actives[RUNCPU]
# r5_bank1 = ker_stack[RUNCPU]
# r6_bank1 = RUNCPU << 2
#
# r7_bank1 is initialised by _start and _smpstart
#
#-------------------------------------------------------------------------

routine_start ker_start, 1

.ifdef	VARIANT_smp
	#
	# Set the ker_stack_top/ker_stack_bot values used by am_inkernel()
	#
	# The statically initialised values in externs.h are a temporary
	# measure to ensure that am_inkernel() works for initialisation code
	# that uses the am_inkernel() macro.
	#
	mov.l	Lks_run_ker_stack_top, r1
	mov.l	Lks_ker_stack_top, r0
	mov.l	@r1, r1
	mov.l	r1, @r0							! ker_stack_top = run_ker_stack_top

	mov.l	Lks_run_ker_stack_bot, r1
	mov.l	Lks_ker_stack_bot, r0
	mov.l	@r1, r1
	mov.l	r1, @r0							! ker_stack_bot = run_ker_stack_bot

	#
	# Set the bank1 registers
	#
	mov.l	Lks_cpidr, r1
	mov.l	@r1, r1
	shll2	r1								! RUNCPU << 2
	ldc		r1, r6_bank

	mov.l	Lks_kerstack, r0
	mov.l	@(r0, r1), r15					! ker_stack[RUNCPU]
	ldc		r15, r5_bank

	mov.l	Lks_actives, r0
	add		r1, r0							! &actives[RUNCPU]
	ldc		r0, r4_bank

	#
	# Set inkernel/cpunum: (RUNCPU << 24) | INKERNEL_LOCK | INKERNEL_NOW
	#
	mov		#22, r0
	shld	r0, r1							! RUNCPU << 24
	mov		r1, r0
	or		#INKERNEL_NOW|INKERNEL_LOCK, r0
	stc		r7_bank, r1
	mov.l	r0, @r1							! set inkernel/cpunum
	bra		__ker_exit
	nop
.else
	#
	# Set the bank1 registers
	#
	mov.l	Lks_kerstack, r0
	mov.l	@r0, r0							! ker_stack[0]
	ldc		r0, r5_bank

	mov		#0, r2
	ldc		r2, r6_bank						! RUNCPU is always 0

	mov.l	Lks_actives, r0					! &actives[0]
	ldc		r0, r4_bank

	bra		__ker_exit
	ldc		r2, r7_bank						! set inkernel = 0
.endif

	.align 2

Lks_actives:			.long	actives
Lks_regoff:				.long	REG_OFF
Lks_kerstack:			.long	ker_stack
.ifdef	VARIANT_smp
Lks_ker_stack_top:		.long	ker_stack_top
Lks_ker_stack_bot:		.long	ker_stack_bot
Lks_run_ker_stack_top:	.long	run_ker_stack_top
Lks_run_ker_stack_bot:	.long	run_ker_stack_bot
Lks_cpidr:				.long	SH4A_MMR_CPIDR
.endif

routine_end ker_start

.ifdef VARIANT_smp
#-------------------------------------------------------------------------
#
# Called on trap/exception entry after context has been saved:
# - usr_acquire_kernel is invoked on entry from user mode
# - sys_acquire_kernel is invoked on entry from kernel/interrupt
#
# On entry:
# - we are executing in bank1 (SR.BL=1, SR.RB=1).
# - r0_bank1 contains the inkernel bits to set.
#
# On return:
# - we are executing in bank0 with interrupts disabled
# - r3_bank1 is preserved
# - r13 contains the old inkernel value
#
# beg_acquire_kernel/end_acquire_kernel mark the range of code intr_done
# checks to figure out if the interrupted context was spinning waiting to
# acquire the kernel:
# - usr_acquire_kernel is preemptable so the syscall is restarted
# - sys_acquire_kernel is not preemptable
#
# WARNING: intr_done case6 checks which bits we are attempting to set
#          by examining the r5 register in the saved context.
#
#          If this code is changed to use a different register, make
#          sure intr_done is in sync.
#
#-------------------------------------------------------------------------
	.align 2

beg_acquire_kernel:

usr_acquire_kernel:
	#
	# Switch to bank0 with interrupts disabled
	#
	ldc		r0, r5_bank				! r5_bank0  = inkernel bits to set
	mov.l	Lak_SR_not_BL_RB, r1
	mov.l	Lak_SR_FD_IMASK, r2
	stc		sr, r0
	and		r1, r0
	or		r2, r0
	ldc		r0, sr

	stc		r7_bank, r1				! r1 = &inkernel
	SMP_GETCPU	r3					! r3 = RUNCPU << 2
	mov		#22, r0
	shld	r0, r3					! r3 = RUNCPU << 24

0:	#
	# Enable interrupts and wait for INKERNEL_NOW to be clear
	# FIXME_INTR: should this set to __shadow_imask?
	#
	mov.l	Lak_SR_not_IMASK, r0
	stc		sr, r2
	and		r0, r2
	ldc		r2, sr

1:
	# FIXME_SMP: need_to_run/need_to_run_cpu stuff here...

	#
	# Wait for INKERNEL_NOW to be cleared
	#
	mov.l	@r1, r0
	tst		#INKERNEL_NOW, r0
	bf		1b
	.word	0x0163					! movli.l @r1, r0
	tst		#INKERNEL_NOW, r0
	bf		1b
	mov		r0, r13					! stash old inkernel value

	#
	# Disable interrupts and attempt to set inkernel/cpunum
	#
	stc		sr, r0
	or		#SH_SR_IMASK, r0
	ldc		r0, sr

	mov		r13, r0
	shll8	r0
	shlr8	r0						! clear cpunum
	or		r5, r0					! set inkernel bits
	or		r3, r0					! set cpunum
	.word	0x0173					! movco.l r0, @r1
	bf		0b						! movco failed - retry whole attempt

	rts
	nop

	.align 2
sys_acquire_kernel:
	#
	# Switch to bank0 in preparation for enabling interrupts
	#
	ldc		r0, r5_bank				! r5  = inkernel bits to set
	mov.l	Lak_SR_not_BL_RB, r1
	mov.l	Lak_SR_FD_IMASK, r2
	stc		sr, r0
	and		r1, r0
	or		r2, r0
	ldc		r0, sr

	stc		r7_bank, r1				! r1 = &inkernel
	SMP_GETCPU	r3					! r3 = RUNCPU << 2
	mov		#22, r0
	shld	r0, r3					! r3 = RUNCPU << 24

0:	#
	# Enable interrupts and check if we own the kernel
	# FIXME_INTR: should this set to __shadow_imask?
	#
	mov.l	Lak_SR_not_IMASK, r0
	stc		sr, r2
	and		r0, r2
	ldc		r2, sr

	mov.l	Lak_cpunum_mask, r2
1:	mov.l	@r1, r0
	tst		#INKERNEL_NOW, r0
	bt		2f						! INKERNEL_NOW is clear, go ahead and try to grab it.
	and		r2, r0
	cmp/eq	r3, r0
	bf		1b						! we don't own the kernel - spin

	# Looks like it's safe to try to grab the kernel.
2:	.word	0x0163					! movli.l @r1, r0
	mov		r0, r13					! stash old inkernel value
	tst		#INKERNEL_NOW, r0
	bt		3f						! INKERNEL_NOW is clear
	and		r2, r0
	cmp/eq	r3, r0
	bf		1b						! somebody else snuck in - spin

3:	#
	# Disable interrupts and attempt to set inkernel
	#
	stc		sr, r0
	or		#SH_SR_IMASK, r0
	ldc		r0, sr

end_acquire_kernel:

	mov		r13, r0
	shll8	r0
	shlr8	r0						! clear cpunum
	or		r5, r0					! set inkernel bits
	or		r3, r0					! set cpunum
	.word	0x0173					! movco.l r0, @r1
	bf		0b						! movco failed - retry whole attempt

	rts
	nop

	.align 2

Lak_SR_not_BL_RB:	.long	~(SH_SR_BL | SH_SR_RB)
Lak_SR_not_IMASK:	.long	~(SH_SR_IMASK)
Lak_cpunum_mask:	.long	0xff000000
Lak_SR_FD_IMASK:	.long	(SH_SR_FD | SH_SR_IMASK)

.endif


#-------------------------------------------------------------------------
#
# Emulation of cmpxchg operation, used to implement atomic_* funcs and
# _smp_cmpxchg()/_smp_xchg() functions via trapa #0xff instruction.
#
# We are executing in bank1 (SR.BL=1, SR.RB=1)
#
# r4_bank1 = &actives[RUNCPU]
#
# cmpxchg r5, @r4:
# 	If r0 == @r4
#	Then T = 1, @r4 = r5 (r0 holds the old value of the location)
#	Else T = 0, r0 = @r4 (r0 holds the changed value of the location)
#		r5 will be destroyed.
# PR is not saved, so should not have any function calls inside this routine.
#
#-------------------------------------------------------------------------

__ker_emu_cmpxchg:
	#
	# Save sr, pr and r1
	#
	mov.l	Lec_emu_save, r1
	SMP_GETCPU_BANK1	r0
	SMP_ADD	r1, r0
	mov.l	@r1, r1						! __ker_emu_save[RUNCPU]
	mov		#-1, r2
	add		#20, r1
	stc.l	r0_bank, @-r1				! save user r0 (used as temporary below)
	stc.l	r1_bank, @-r1				! save user r1 (used as temporary below)
	stc		ssr, r0
	or		#1, r0						! set SR.T bit (assume we'll succeed)
	mov.l	r0, @-r1					! save user SR
	stc.l	spc, @-r1					! save user PR
	mov.l	r2, @-r1					! set active indicator

	#
	# Check address in r4_bank0 is valid ( <= act->process->boundry_addr
	#
	mov.l	@r4, r0
	mov.l	Lec_PROCESS, r1
	mov.l	Lec_BOUNDRY_ADDR, r2
	add		r0, r1
	mov.l	@r1, r0						! r0 = act->process
	tst		r0, r0
	bt		0f
	add		r0, r2
	mov.l	@r2, r1						! r1 = act->process->boundry_addr
	stc		r4_bank, r0
	cmp/hi	r0, r1
	bf		__ker_emu_cmpxchg_fault


	#
	# Disable interrupts, clear BL and switch to bank0
	#
0:	mov.l	Lec_SR_not_BL_RB, r2
	stc		sr, r0
	and		r2, r0
	or		#SH_SR_IMASK, r0
	ldc		r0, sr

	#
	# Now using bank0 registers
	#
	# On entry:
	# r0 - original value of location (set by user code before trapa call)
	# r4 - address of location
	# r5 - new value to set
	#
	# On return:
	# r0 - current value of location
	# r5 - trashed
	# sr - T bit set if operation succeeded
	#
	# Note that we can take exceptions here - the exception entry code
	# checks __ker_emu_save[RUNCPU] to figure out if we were in this code
	# and if so, it will restore the context from __ker_emu_save[]
	#
	mov.l	@r4, r1
	cmp/eq	r0, r1
	bf		1f

	#
	# compare succeeded - set new value and restore context
	# We don't restore r0 since it was unchanged
	#
	mov.l	r5, @r4

	mov.l	Lec_emu_save, r1
	SMP_GETCPU	r5
	SMP_ADD	r1, r5
	mov.l	@r1, r1					! __ker_emu_save[RUNCPU]
	mov		#0, r5
	mov.l	r5, @r1					! clear active indicator
	add		#4, r1
	ldc.l	@r1+,spc				! set user pc
	ldc.l	@r1+,ssr				! restore user sr (T bit set)
	mov.l	@r1, r1					! set user r1
	rte
	nop

	#
	# cmpxchg failed - set user r0 to current value and restore context
	#
1:	mov		r1, r0					! user r0 = current value of location

	mov.l	Lec_emu_save, r1
	SMP_GETCPU	r5
	SMP_ADD	r1, r5
	mov.l	@r1, r1					! __ker_emu_save[RUNCPU]
	mov		#0, r5
	mov.l	r5, @r1					! clear active indicator
	add		#4, r1
	ldc.l	@r1+,spc				! set user pc
	mov.l	@r1+, r5
	add		#-1, r5					! clear SR.T bit
	ldc		r5,ssr					! set user sr (T bit clear)
	mov.l	@r1, r1					! set user r1
	rte
	nop

__ker_emu_cmpxchg_fault:
	#
	# Address is not valid:
	# - save context
	# - acquire kernel
	# - switch to bank0 with interrupts and FPU disabled
	#
	EXC_SAVECONTEXT
	EXC_ENTERKERNEL

	mov		r4, r6
	mov.l	Lec_SIGSEGV_code, r4
	mov.l	Lec_exc, r0
	jmp		@r0
	nop

	.align 2

Lec_SR_not_BL_RB:	.long	~(SH_SR_BL | SH_SR_RB)
Lec_emu_save:		.long	__ker_emu_save
Lec_PROCESS:		.long	PROCESS
Lec_BOUNDRY_ADDR:	.long	BOUNDRY_ADDR
Lec_SIGSEGV_code:	.long	SIGSEGV+ (SEGV_ACCERR*256) + (FLTBOUNDS*65536)
Lec_exc:			.long	__exc


#-------------------------------------------------------------------------
#
# Code for handling illegal system call numbers, called by __ker_entry
#
# We are executing in bank1 (SR.BL=1, SR.RB=1)
#
# r3 contains TRA << 2
#
#-------------------------------------------------------------------------

bad_func:
	#
	# Check for special trap numbers used for various emulation operations
	#
	mov.l	Lbf_SH_KER_TRAP_BOUNDARY, r0
	cmp/hs	r0, r3
	bt		bad_func_emulation

	#
	# Bad system call number:
	# - set user r0 to ENOSYS
	# - advance user pc to error handling code in system call stub
	#
	mov		#ENOSYS, r0
	ldc		r0, r0_bank
	stc		spc, r0
	add		#KERERR_SKIPAHEAD, r0
	ldc		r0,spc
	rte
	nop

	.align 2

Lbf_SH_KER_TRAP_BOUNDARY:	.long (SH_KER_TRAP_BOUNDARY<<2)

bad_func_emulation:
	sub		r0, r3
	mova	bad_func_handler_table, r0
	mov.l	@(r0, r3), r2
	jmp		@r2
	nop

	.align 2

	# SH_KER_TRAP_BOUNDARY is 0xfd
	# trapa #0xfd -> __inline_DebugKDOutput
	# trapa #0xfe -> __inline_DebugBreak
	# trapa #0xff -> cmpxchg emulation
	# Note: if this table changes, you must also change ker/sh/kercpu.h 
	# constants SH_KER_TRAP_*.
bad_func_handler_table:
	.long	__handle_kdbmsg
	.long	__exc_usr_break_enter
	.long	__ker_emu_cmpxchg


#-------------------------------------------------------------------------
#
# System call entry point (trapa instruction)
#
# We are executing in bank 1 (SR.BL=1, SR.RB=1)
#
# r4_bank1 = &actives[RUNCPU]
# r5_bank1 = ker_stack[RUNCPU]
# r6_bank1 = RUNCPU << 2
# r7_bank1 = inkernel (non-SMP), &inkernel (SMP)
#
#-------------------------------------------------------------------------

	.align 5

routine_start __ker_entry, 1
	#
	# Check for invalid syscall number:
	# - special trapa values invoke custom code above
	# - illegal system call values set ENOSYS and advance the return pc
	#   to the syscall error handling code in the system call stub
	#
	# Note that TRA register contains the trapa value << 2
	#
	mov.l	Lke_SH_MMR_CCN_TRA, r2
	mov.l	Lke_KER_BAD, r0
	mov.l	@r2, r3
	cmp/hs	r0, r3
	bt		bad_func

	#
	# Save register context
	#
	mov.l	Lke_REG_OFF_END, r0
	mov.l	@r4, r1					! r2 = actives[RUNCPU]
	add		r0, r1					! &act->reg + SIZEOF_REG
	SAVECONTEXT

	mov		r5, r15					! r15 = ker_stack[RUNCPU]
	mov.l	@r4, r12				! r12 = actives[RUNCPU]
	mov		r1, r11					! r11 = saved context
	mov		r3, r8					! r8  = syscall number << 2

	#
	# Acquire the kernel (set INKERNEL_NOW)
	#
.ifdef	VARIANT_smp
	mov.l	Lke_usr_acquire_kernel, r1
	mov		#0, r0
	jsr		@r1
	or		#INKERNEL_NOW, r0
.else
	mov		r7, r0
	mov		r7, r13
	or		#INKERNEL_NOW, r0
	mov		r0, r7
.endif

	#
	# Enable interrupts and switch to bank0
	#
	# We also disable the FPU here - if the kernel uses any FPU operations
	# this will cause an FPU disabled exception that will save any active
	# user FPU context.
	#
	mov.l	Lke_SR_not_RB_BL_FD_IMASK, r1
	mov.l	Lke_SR_FD, r10
	stc		sr, r0
	and		r1, r0
	or		r10, r0
	ldc		r0, sr

	#
	# act->flags &= ~(_NTO_TF_KERERR_SET|_NTO_TF_BUFF_MSG)
	#
	mov.l	Lke_TFLAGS, r0
	mov.l	Lke_TF_mask, r3
	mov.l	@(r0, r12), r2
	and		r3, r2
	mov.l	r2, @(r0, r12)

	#
	# call call_table[syscall_number](args)
	#
.ifdef VARIANT_instr
	mov.l	Lke_trace_call_table, r1
.else
	mov.l	Lke_ker_call_table, r1
.endif
	add		r8, r1
	mov		#REG_R4, r5				! offset of r4 in saved context
	shlr2	r8
	mov.l	@r1, r1					! call_table[syscall_number]
	mov.l	r8, @(SYSCALL, r12)		! set act->syscall
	mov		r12, r4
	jsr		@r1
	add		r11, r5					! start of syscall args &act->reg.gpr[R4]

	#
	# Check syscall return code:
	# ENOERROR: goto __ker_exit (error indication has already been set)
	# 0:        set syscall return register to 0
	# > 0:      call kererr() to set error indication
	#
	cmp/pz	r0
	bf		__ker_exit				! r0 == ENOERROR
	tst		r0, r0
	bf		__set_err

	#
	# Set act->reg r0 to 0
	#
	# WARNING: we don't need to lock kernel here because a restart will not
	#          cause any register corruption:
	#          - syscall number is encoded in the instruction
	#          - syscall args and return value registers do not overlap
	#
	mov.l	Lke_REG_OFF_R0, r1
	add		r12, r1					! &act->reg.gpr[R0]
	bra		__ker_exit
	mov.l	r0, @r1					! set act->reg r0 register

__set_err:
	#
	# Call kererr(act, error)
	#
	mov.l	Lke_kererr, r1
	mov		r0, r5
	jsr		@r1
	mov		r12, r4

	#
	# Fall through to __ker_exit.
	#
routine_end __ker_entry


#-------------------------------------------------------------------------
#
# __ker_exit: return from kernel processing
#
# r4_bank1 = &actives[RUNCPU]
# r5_bank1 = ker_stack[RUNCPU]
# r6_bank1 = RUNCPU << 2
# r7_bank1 = inkernel (non-SMP), &inkernel (SMP)
#
# Within the __ker_exit code path:
# r11 = &act->reg
# r12 = act
# r14 = RUNCPU << 12
#
#-------------------------------------------------------------------------

routine_start __ker_exit, 1
	stc		r4_bank, r1					! &actives[RUNCPU]
	mov.l	Lke_REG_OFF, r11
	mov.l	@r1, r12					! r12 = actives[RUNCPU]
.ifdef	VARIANT_smp
	stc		r7_bank, r1
0:	.word	0x0163						! movli.l @r1, r0
 	or		#(INKERNEL_NOW+INKERNEL_LOCK+INKERNEL_EXIT), r0
	.word	0x173						! movco.l r0, @r1
	bf		0b
.else
	stc		r7_bank, r0
 	or		#(INKERNEL_NOW+INKERNEL_LOCK+INKERNEL_EXIT), r0
	ldc		r0, r7_bank
.endif
	add		r12, r11					! r11 = &act->reg

	#
	# Make sure interrupts are enabled.
	# intr_done calls __ker_exit with interrupts disabled
	#
	mov.l	Lke_SR_not_BL_IMASK, r1
	stc		sr, r2
	and		r1, r2
	ldc		r2, sr

	#
	# Check for any pending events.
	#
	mov.l	Lke_intrevent_pending, r1
	mov.l	@r1, r0
	tst		r0, r0
	bt		0f

	#
	# Drain pending interrupt events
	#
__ker_intrevent:
	#
	# Make sure interrupts are enabled.
	# Code at the end of __ker_exit branches here with interrupts disabled
	#
	mov.l	Lke_SR_not_BL_IMASK, r1
	stc		sr, r0
	and		r1, r0
	mov.l	Lke_intrevent_drain, r2
	ldc		r0, sr
	jsr		@r2
	nop
	bra		__ker_exit
	nop

0:	#
	# Set up common registers used in code below
	#
	mov.l	Lke_actives_prp, r8
	SMP_GETCPU	r14
	SMP_ADD	r8, r14						! r8 = &actives_prp[RUNCPU]
	mov.l	Lke_PROCESS, r1
	mov.l	@r8, r10					! r10 = actives_prp[RUNCPU]
	add		r12, r1
	mov.l	@r1, r9						! r9 = act->process

	#
	# Check for a process switch since we may need to remove breakpoints:
	#
	# actives_prp[RUNCPU] is the currently active (possibly outgoing) process.
	#
	#	if (act->process != actives_prp[RUNCPU) {
	#		if (actives_prp[RUNCPU] && actives_prp[RUNCPU]->debugger) {
	#			if (actives_prp[RUNCPU] != aspaces_prp[RUNCPU]) {
	#				memmgr.aspace(actives_prp[RUNCPU], &aspaces_prp[RUNCPU]);
	#			}
	#			debug_detach_brkpts(actives_prp[RUNCPU]->debugger);
	#		}
	#	}
	#
	cmp/eq	r9, r10
	bt		__ker_aspace_check
	tst		r10, r10
	bt		__ker_aspace_check

	mov.l	Lke_DEBUGGER, r0
	mov.l	Lke_aspaces_prp, r5
	mov.l	@(r0, r10), r4				! actives_prp[RUNCPU]->debugger
	SMP_ADD	r5, r14						! &aspaces_prp[RUNCPU]
	tst		r4, r4
	bt		__ker_aspace_check
	mov.l	@r5, r6						! aspaces_prp[RUNCPU]
	cmp/eq	r10, r6
	bt		0f

	mov.l	Lke_MEMMGR_ASPACE, r0
	mov.l	@r0, r0
	jsr		@r0							! call memmgr.aspace()
	mov		r10, r4						! actives_prp[RUNCPU]

	mov.l	Lke_DEBUGGER, r0			! r4 trashed by function call
	mov.l	@(r0, r10), r4				! reload actives_prp[RUNCPU]->debugger

0:	mov.l	Lke_debug_detach_brkpts, r1
	mov.l	@r1, r0						! call debug_detach_brkpts()
	jsr		@r0							! remove old breakpoint
	nop

__ker_aspace_check:
	#
	# Check for a aspace switch:
	#
	#	if (act->aspace_prp != aspaces_prp[RUNCPU]) {
	#		if (act->aspace_prp) {
	#			memmgr.aspace(act->aspace_prp, &aspaces_prp[RUNCPU]);
	#		}
	#	}
	#
	mov.l	Lke_ASPACE_PRP, r1
	mov.l	Lke_aspaces_prp, r5
	add		r12, r1
	SMP_ADD	r5, r14						! &aspaces_prp[RUNCPU]
	mov.l	@r1, r0						! act->aspace_prp
	mov.l	@r5, r1						! aspaces_prp[RUNCPU]
	cmp/eq	r1, r0
	bt		__ker_prp_check
	cmp/eq	#0, r0
	bt		__ker_prp_check

	#
	# Switch address space
	#
	mov.l	Lke_MEMMGR_ASPACE, r1
	mov.l	@r1, r1
	jsr		@r1
	mov		r0, r4						! act->aspace_prp

__ker_prp_check:
	#
	# Check for a process switch:
	#
	#	if (act->process != actives_prp[RUNCPU]) {
	#		actives_prp[RUNCPU] = act->process;
	#		cpupageptr[RUNCPU]->pls = act->process->pls;
	#		if (act->process->debugger) {
	#			debug_attach_brkpts(act->process->debugger);
	#		}
	#	}
	#
	cmp/eq	r9, r10
	bt		__ker_specret_check

	mov.l	Lke_cpupageptr, r1
	mov.l	r9, @r8						! actives_prp[RUNCPU] = act->process
	SMP_ADD	r1, r14
	mov.l	Lke_PLS, r0
	mov.l	@r1, r2						! cpupageptr[RUNCPU]
	mov.l	Lke_CPUPAGE_PLS, r1
	mov.l	@(r0, r9), r3				! act->process->pls
	mov.l	Lke_DEBUGGER, r0
	add		r2, r1						! &cpupageptr[RUNCPU]->pls
	mov.l	@(r0, r9), r4				! act->process->debugger
	mov.l	r3, @r1						! set cpupageptr[RUNCPU]->pls

	#
	# Check if we need to set breakpoints
	#
	tst		r4, r4
	bt		__ker_specret_check
	mov.l	Lke_debug_attach_brkpts, r1
	mov.l	@r1, r0
	jsr		@r0
	nop

__ker_specret_check:
	#
	# Check for special actions
	#
	mov.l	Lke_TFLAGS, r1
	mov.l	Lke_SPECRET_MASK, r2
	add		r12, r1
	mov.l	@r1, r0
	tst		r2, r0
	bf		__ker_specret

__ker_fpu_disable:
	#
	# Disable FPU if necessary so we can perform a lazy context switch
	#
	mov.l	Lke_actives_fpu, r1
	mov.l	Lke_SR_FD, r2
	SMP_ADD	r1, r14						! &actives_fpu[RUNCPU]
	mov.l	@r1, r1
	mov		#REG_SR, r0
	cmp/eq	r1, r12
	bt.s	1f
	mov.l	@(r0, r11), r1				! saved SR

	#
	# act != actives_fpu[RUNCPU]
	# Disable FPU so we can lazily switch context
	#
	bra		2f
	or		r2, r1						! disable FPU 

1:	#
	# act == actives_fpu[RUNCPU]
	# We can simply enable the FPU since it hasn't been touched by kernel/ISR
	#
	not		r2, r2
	and		r2, r1						! enable FPU

2:	mov.l	r1, @(r0, r11)

.ifdef VARIANT_instr
	#
	# Check for kernel exit tracing
	#
	mov.l	Lke_ker_exit_enable_mask, r1
	mov.l	@r1, r0
	tst		r0, r0
	bt		1f

	#
	# Call ker_trace_exit(act)
	#
	mov.l	Lke_trace_ker_exit, r0
	jsr		@r0
	mov		r12, r4
1:
.endif

	#
	# Check for mutex atomic operation.
	#
	# If we were inside the critical region (stack pointer has bit 0 set)
	# call cpu_mutex_adjust() to rewind the user code to retry the operation.
	#
	mov.l	Lke_REG_OFF_R15, r1
	add		r12, r1
	mov.l	@r1, r0
	tst		#1, r0
	bt		1f
	mov.l	Lke_cpu_mutex_adjust, r0
	jsr		@r0
	mov		r12, r4
1:

	#
	# Disable interrupts and perform a last check for pending interrupt events
	#
	stc		sr, r0
	mov.l	Lke_intrevent_pending, r1
	or		#SH_SR_IMASK, r0
	ldc		r0, sr

	mov.l	@r1, r0
	tst		r0, r0
	bf		__ker_intrevent

	#
	# Check act->async_flags
	#
	mov.l	Lke_ATFLAGS, r1
	add		r12, r1
.ifdef	VARIANT_smp
0:	.word	0x0163				! movli.l @r1, r0
	mov		r0, r2
	mov		#0, r0
	.word	0x0173				! movco.l r0, @r1
	bf		0b
.else
	mov.l	@r1, r2
	mov.l	r0, @r1				! we know r0 == 0
.endif
	cmp/eq	r0, r2
	bf		__ker_atflags

	#
	# Set cpupageptr[RUNCPU]->tls to act->tls
	#
	mov.l	Lke_TLS, r1
	mov.l	Lke_cpupageptr, r4
	SMP_ADD	r4, r14				! r4 = &cpupageptr[RUNCPU]
	add		r12, r1
	mov.l	@r1, r2
	mov.l	@r4, r3
	mov.l	Lke_CPUPAGE_TLS, r1
	add		r3, r1
	mov.l	r2, @r1

.ifdef	VARIANT_smp
	stc		r7_bank, r1
	mov.l	Lke_not_INKERNEL_bits, r2
0:	.word	0x0163				! movli.l @r1, r0
	and		r2, r0				! clear all inkernel bits
	.word	0x0173				! movco.l r0, @r1
	bf		0b
.else
	ldc		r0, r7_bank			! r0 is 0 from above
.endif

.ifdef	VARIANT_smp
	#
	# intr_done branches here if:
	#
	# 1) kernel is locked or in specialret on another cpu (case2)
	#    All we can do is restore the interrupted context.
	#    The other cpu will presumably handle any rescheduling when it exits
	#    the kernel.
	#
	# 2) queued_event_priority requires rescheduling to occur (case2)
	#    We have sent an ipi to that cpu to check for rescheduling, so we
	#    simply restore the interrupted context.
	#
	# 3) we were spinning waiting to acquire the kernel, but the kernel is
	#    still held by another cpu (case6 -> case 2)
	#    For system calls, we have backed up the pc to restart the syscall.
	#    For other exceptions, we just restore the context to re-trigger the
	#    exception from user mode.
	#
	# In all of these cases, we still haven't processed any pending interrupt
	# events, se we send an IPI to another cpu to process them since we can't
	# handle them at this point.
	#
__ker_exit2:

	#
	# FIXME_SMP: check need_to_run/need_to_run_cpu
	#

	#
	# Check if we need to send an IPI to another cpu to handle intrevents.
	#
	mov.l	Lke_intrevent_pending, r0
	mov.l	@r0, r0
	tst		r0, r0
	bt		1f

	#
	# Send IPI to (RUNCPU + 1) % num_processors
	#
	mov.l	Lke_num_processors, r0
	stc		r6_bank, r4					! RUNCPU << 2
	shlr2	r4							! RUNCPU
	mov.l	@r0, r0
	add		#1, r4
	cmp/eq	r4, r0
	bf		0f
	mov		#0, r4
0:	mov.l	Lke_send_ipi, r0
	mov		#IPI_CHECK_INTR, r5
	jsr		@r0
	nop

1:
.endif

__ker_iret:
	#
	# Restore context (r11 points to base of saved context)
	#
	mov.l	@r11, r0
	mov.l	@(4, r11), r1
	mov.l	@(8, r11), r2
	mov.l	@(12, r11), r3
	mov.l	@(16, r11), r4
	mov.l	@(20, r11), r5
	mov.l	@(24, r11), r6
	mov.l	@(28, r11), r7
	mov.l	@(32, r11), r8
	mov.l	@(36, r11), r9
	mov.l	@(40, r11), r10
	mov.l	@(48, r11), r12
	mov.l	@(52, r11), r13
	mov.l	@(56, r11), r14
	mov.l	@(60, r11), r15
	add		#64, r11
	ldc.l	@r11+,ssr
	ldc.l	@r11+,spc
	ldc.l	@r11+,gbr
	lds.l	@r11+,mach
	lds.l	@r11+,macl
	lds.l	@r11+,pr
	add		#-44, r11
	mov.l	@r11, r11
	rte
	# WARNING: do not optimize this slot, may cause privilege problems
	nop

#
# __ker_exit specialret() processing
#
__ker_specret:
	stc		r5_bank, r15			! Reset kernel stack

.ifdef	VARIANT_smp
	#
	# Restore saved registers if _NTO_ATF_FORCED_KERNEL was set
	#
	stc		sr, r0
	or		#SH_SR_IMASK, r0
	ldc		r0, sr

	mov.l	Lke_ATFLAGS, r1
	mov.l	Lke_ATF_FORCED_KERNEL, r2
	add		r12, r1						! &act->async_flags
	not		r2, r3
0:	.word	0x0163						! movli.l @r1, r0
	tst		r2, r0
	bt		1f
	and		r3, r0						! clear _NTO_ATF_FORCED_KERNEL bit
	.word	0x0173						! movco.l r0, @r1
	bf		0b

	#
	# Since the system call number is in the trapa instruction, we only
	# have to save/restore the pc register.
	#
	mov.l	Lke_ARGS_ASYNC_IP, r1
	mov.l	Lke_REG_OFF_PC, r0
	add		r12, r1					! &act->args.async.save_ip
	mov.l	@r1, r2					! pick up saved pc
	mov.l	r2, @(r0, r12)			! put back in act->reg.pc

1:	mov.l	Lke_SR_not_BL_IMASK, r1
	stc		sr, r0
	and		r1, r0					! re-enable interrupts
	ldc		r0, sr
.endif

	#
	# Call specialret() then go back to __ker_exit in case actives[] changed
	#
	mov.l	Lke_specialret, r0
	jsr		@r0
	mov		r12, r4
	bra		__ker_exit
	nop

#
# __ker_exit async_flags processing
#
#	_NTO_ATF_FORCED_KERNEL:	restore registers before forced syscall
#	_NTO_ATF_FPUSAVE_ALLOC: allocate FPU context save area
#	_NTO_ATF_TIMESLICE:     call reschedule()
#	_NTO_ATF_SMP_RESCHED:   call reschedule()
#
# r2 contains the pending flags to process
#
__ker_atflags:
.ifdef	VARIANT_smp
	#
	# Restore saved registers if _NTO_ATF_FORCED_KERNEL was set
	#
	mov.l	Lke_ATF_FORCED_KERNEL, r3
	tst		r3, r2
	bt		1f

	#
	# Restore registers
	#
	# Since the system call number is in the trapa instruction, we only
	# have to save/restore the pc register.
	#
	mov.l	Lke_ARGS_ASYNC_IP, r1
	mov.l	Lke_REG_OFF_PC, r0
	add		r12, r1					! &act->args.async.save_ip
	mov.l	@r1, r3					! pick up saved pc
	mov.l	r3, @(r0, r12)			! put back in act->reg.pc
1:
.endif

	#
	# Enable interrupts
	#
	mov.l	Lke_SR_not_BL_IMASK, r1
	stc		sr, r0
	and		r1, r0
	ldc		r0, sr

	mov		#_NTO_ATF_FPUSAVE_ALLOC, r0
	tst		r2, r0
	bt		1f
	mov.l	Lke_fpusave_alloc, r0
	jsr		@r0
	nop
	bra		__ker_exit
	nop

1:	mov		#_NTO_ATF_TIMESLICE|_NTO_ATF_SMP_RESCHED, r0
	tst		r2, r0
	bt		1f
	mov.l	Lke_resched, r1
	mov.l	@r1, r0
	jsr		@r0
	nop

1:	bra		__ker_exit
	nop

	.align	2

Lke_SH_MMR_CCN_TRA:			.long	SH_MMR_CCN_TRA
Lke_KER_BAD:				.long	__KER_BAD<<2
Lke_SR_not_RB_BL_FD_IMASK:	.long	~(SH_SR_RB | SH_SR_BL | SH_SR_FD | SH_SR_IMASK)
Lke_SR_FD:					.long	SH_SR_FD
Lke_TFLAGS:					.long	TFLAGS
Lke_TF_mask:				.long	~(_NTO_TF_KERERR_SET | _NTO_TF_BUFF_MSG)
Lke_SR_not_BL_IMASK:		.long	~(SH_SR_BL | SH_SR_IMASK)
Lke_PROCESS:				.long	PROCESS
Lke_ASPACE_PRP:				.long	ASPACE_PRP
Lke_SPECRET_MASK:			.long	_NTO_TF_SPECRET_MASK
Lke_ATFLAGS:				.long	ATFLAGS
Lke_TLS:					.long	TLS
Lke_PLS:					.long	PLS
Lke_DEBUGGER:				.long	DEBUGGER
Lke_REG_OFF:				.long	REG_OFF
Lke_REG_OFF_END:			.long	REG_OFF+SIZEOF_REG
Lke_REG_OFF_R0:				.long	REG_OFF+REG_R0
Lke_REG_OFF_R15:			.long	REG_OFF+REG_R15
Lke_cpupageptr:				.long	cpupageptr
Lke_CPUPAGE_TLS:			.long	CPUPAGE_TLS
Lke_CPUPAGE_PLS:			.long	CPUPAGE_PLS
Lke_MEMMGR_ASPACE:			.long	MEMMGR_ASPACE + memmgr
Lke_cpu_mutex_adjust:		.long	cpu_mutex_adjust
Lke_intrevent_pending:		.long	intrevent_pending
Lke_intrevent_drain:		.long	intrevent_drain
Lke_actives_prp:			.long	actives_prp
Lke_aspaces_prp:			.long	aspaces_prp
Lke_kererr:					.long	kererr
Lke_debug_detach_brkpts:	.long	debug_detach_brkpts
Lke_debug_attach_brkpts:	.long	debug_attach_brkpts
Lke_ker_call_table:			.long	ker_call_table
Lke_specialret:				.long	specialret
Lke_fpusave_alloc:			.long	fpusave_alloc
Lke_resched:				.long	resched
.ifdef	VARIANT_instr
Lke_trace_call_table:		.long	_trace_call_table
Lke_ker_exit_enable_mask:	.long	ker_exit_enable_mask
Lke_trace_ker_exit:			.long	_trace_ker_exit
.endif
.ifdef	VARIANT_smp
Lke_usr_acquire_kernel:		.long	usr_acquire_kernel
Lke_num_processors:			.long	num_processors
Lke_not_INKERNEL_bits:		.long	~(INKERNEL_NOW|INKERNEL_LOCK|INKERNEL_SPECRET|INKERNEL_EXIT)
Lke_send_ipi:				.long	send_ipi
Lke_ATF_FORCED_KERNEL:		.long	_NTO_ATF_FORCED_KERNEL
Lke_REG_OFF_PC:				.long	REG_OFF+REG_PC
Lke_ARGS_ASYNC_IP:			.long	ARGS_ASYNC_IP
.endif
Lke_actives_fpu:			.long	actives_fpu

routine_end __ker_exit

#-------------------------------------------------------------------------
#
# Interrupt Handling.
#
# The code sequence between intr_entry_start and intr_entry_end is used
# by cpu_interrupt_init to mix with startup provided code bursts on the
# fly to detect the interrupt source.
#
# Once the interrupt source has been identified, intr_process_queue is
# invoked to dispatch the interrupt to the appropriate handlers, and on
# return from interrupt handling, intr_done is invoked to return from the
# interrupt.
#
#-------------------------------------------------------------------------

	.globl	intr_entry_start
	.align 2
intr_entry_start:
	#
	# We are executing in bank1 (SR.BL=1, SR.RB=1)
	#
.ifdef	VARIANT_smp
	#
	# We can't check inkernel on SMP since another cpu may have the kernel.
	# Instead we check if ker_stack_bot < r15 <= ker_stack_top
	#
	mov.l	Lie_ker_stack_bot, r0
	mov.l	Lie_ker_stack_top, r1
	mov.l	@r0, r0
	mov.l	@r1, r1
	cmp/hs	r0, r15						! is r15 > ker_stack_bot?
	bf		intr_entry_user
	cmp/hi	r1, r15						! is r15 > ker_stack_top?
	bt		intr_entry_user
.else
	tst		r7, r7
	bt		intr_entry_user
.endif

	#
	# Already on kernel stack
	mov		r15, r1
	SAVECONTEXT
	mov		r1, r15
	mov		r1, r11						! r11 = saved context
	mov.l	@r4, r12					! actives[RUNCPU]
	bra		2f
	mov		#1, r13						! tell intr_done we're from kernel

intr_entry_user:
	#
	# Save context in act->reg
	#
	mov.l	@r4, r2						! actives[RUNCPU]
	mov.l	Lie_REG_OFF_END, r0
	mov		r2, r1
	add		r0, r1
	SAVECONTEXT
	mov		r5, r15						! ker_stack[RUNCPU]
	mov		r1, r11						! r11 = saved context
	mov		r2, r12						! r12 = actives[RUNCPU]
	mov		#0, r13						! tell intr_done we're from user

2:
	#
	# Increment the inkernel interrupt nesting counter
	#
.ifdef	VARIANT_smp
0:	.word	0x0763						! movli.l @r7, r0
	add		#1, r0
	.word	0x0773						! movco.l r0, @r7
	bf		0b
.else
	add		#1, r7
.endif

	#
	# Switch to bank0 with interrupts enabled but masked.
	# We also need to disable the FPU in case kernel/ISR uses the FPU.
	#
	# WARNING: interrupt id callouts will change the SR.IMASK based on the
	#          assigned interrupt priority.
	#          The __shadow_imask variable is used to track this priority mask.
	#          See interrupt.c:interrupt() and __inline_InterruptEnable().
	#
	mov.l	Lie_SR_not_RB_BL_FD, r1
	mov.l	Lie_SR_FD, r2
	stc		sr, r0
	and		r1, r0
	or		#SH_SR_IMASK, r0
	or		r2, r0
	ldc		r0, sr

	ACQUIRE_INTR_SLOCK	Lie_intr_slock

	bra		intr_entry_end		! jump to dynamically generated interrupt code
	nop

	.align 2
Lie_REG_OFF_END:		.long	REG_OFF+SIZEOF_REG
Lie_SR_not_RB_BL_FD:	.long	~(SH_SR_RB|SH_SR_BL|SH_SR_FD)
Lie_SR_FD:				.long	SH_SR_FD
.ifdef	VARIANT_smp
Lie_ker_stack_bot:		.long	ker_stack_bot
Lie_ker_stack_top:		.long	ker_stack_top
Lie_intr_slock:			.long	intr_slock
.endif

intr_entry_end:
	 .type intr_entry_start, @function
	 .type intr_entry_end, @function
#-------------------------------------------------------------------------
# End of copied interrupt code
#-------------------------------------------------------------------------

#--------------------------------------------------------------------------
# intr_process_queue
#
# This code fragment is branched to by the interrupt dispatch code
# generated by cpu_interrupt_init().
#
# r4 already contains ilp
#--------------------------------------------------------------------------

routine_start intr_process_queue, 1
#	Call interrupt(ilp, was_inkernel, reg_context_p)
	mov		r13, r5	! was_inkernel, parameter 2
	mov		r11, r6 ! register context, parameter 3
.ifdef	VARIANT_smp
	RELEASE_INTR_SLOCK	Lid_intr_slock
	mov.l	Lid_interrupt, r0
	sts.l	pr, @-r15
	jsr		@r0
	nop
	ACQUIRE_INTR_SLOCK	Lid_intr_slock
	lds.l	@r15+, pr
	rts
	nop
.else
	mov.l	Lid_interrupt, r0
	jmp		@r0
	nop
.endif

routine_end	intr_process_queue


#--------------------------------------------------------------------------
#
# intr_done_chk_fault
#
# This code fragment is branched to by the interrupt dispatch code
# generated by cpu_interrupt_init() to return from an interrupt that
# causes a cpu exception.
#
#	All done dispatching this interrupt. An EOI has been sent to the
#	interrupt controller by the startup/generated code. Check to see
#	if a cpu exception has occurred and handle it if so. If not,
#	figure out how to continue the interrupted context.
#	Interupts are disabled.
#
#--------------------------------------------------------------------------

routine_start intr_done_chk_fault, 1
	cmp/eq	#0, r0						! any exception?
	bt		intr_done

	#
	# Atomically decrement inkernel interrupt count and handle exception
	# We assume the callout code has set:
	# r4 to the exception code
	# r6 to the exception address (if relevant)
	#
.ifdef	VARIANT_smp
	RELEASE_INTR_SLOCK	Lid_intr_slock
	stc		r7_bank, r1
0:	.word	0x0163				! movli.l @r1, r0
	add		#-1, r0
	.word	0x0173				! movco.l r0, @r1
	bf		0b
.else
	stc		r7_bank, r0
	add		#-1, r0
	ldc		r0, r7_bank
.endif
	bra		__exc
	nop

routine_end	intr_done_chk_fault

#--------------------------------------------------------------------------
#
# intr_done
#
# This code fragment is branched to by the interrupt dispatch code
# generated by cpu_interrupt_init() to return from interrupt handling
#
# We have 6 cases to consider:
#          This CPU      Another CPU    Action
#          --------      -----------    ------
# case1    from user     in user        Become the kernel
# case2    from user     in kernel      Check preempt and maybe ipi (SMP)
# case3    from kernel   in usr         Check preempt and maybe become kernel
# case4    from intr     in user        Return to user
# case5    from intr     in kernel      Return to user (SMP)
# case6    kacquire spin *              Try to become kernel, else return (SMP)
#
# On entry:
# - interrupts are disabled
# - r11 = saved context
# - r13 = 1 if entered from kernel, 0 if entered from user
#
#--------------------------------------------------------------------------

routine_start intr_done, 1
	#
	# r12 is trashed by generated interrupt code, so restore it
	#
	stc		r4_bank, r1
	mov.l	@r1, r12			! actives[RUNCPU]

	#
	# Decrement the interrupt nesting counter and determine how to return
	#
.ifdef	VARIANT_smp
	RELEASE_INTR_SLOCK	Lid_intr_slock
	stc		r7_bank, r1
0:	.word	0x0163				! movli.l @r1, r0
	add		#-1, r0
	.word	0x0173				! movco.l r0, @r1
	bf		0b
	tst		r13, r13
	bf		from_kerorintr
.else
	stc		r7_bank, r0
	dt		r0
	ldc		r0, r7_bank
	bf		from_kerorintr
.endif

	#
	# Interrupted user code
	#
case1:
.ifdef	VARIANT_smp
	#
	# Try to acquire the kernel and return via __ker_exit
	#
	mov		#22, r2
	SMP_GETCPU	r3							! RUNCPU << 2
	shld	r2, r3							! RUNCPU << 24

	stc		r7_bank, r1
0:	.word	0x0163							! movli.l @r1, r0
	tst		#INKERNEL_NOW|INKERNEL_LOCK, r0
	bf		case2							! another cpu has the kernel
	shll8	r0
	shlr8	r0								! clear cpunum
	or		r3, r0							! set cpunum
	or		#INKERNEL_NOW|INKERNEL_LOCK, r0
	.word	0x0173							! movco.l r0, @r1
	bf		0b
.endif
	bra		__ker_exit
	nop

.ifdef	VARIANT_smp
case2:
	#
	# FIXME: describe this...
	# r0 contains current inkernel bits from case1 code above
	#
	tst		#INKERNEL_LOCK|INKERNEL_SPECRET, r0
	bf		0f								! goto __ker_exit2

	mov		r0, r4							! preserve inkernel for below
	mov.l	Lid_queued_event_priority, r2
	mov.l	Lid_PRIORITY, r0
	mov.l	@r2, r2							! queued_event_priority
	mov.b	@(r0, r12), r0					! actives[RUNCPU]->priority
	and		#0xff,r0
	cmp/hi	r0, r2
	bf		0f								! goto __ker_exit2

	#
	# queued_event_priority > act->priority so send an IPI to the cpu that
	# owns the kernel so it can perform a reschedule
	#
	mov.l	Lid_send_ipi, r0
	shlr16	r4								! r4 has inkernel from above
	shlr8	r4								! cpunum (inkernel >> 24)
	jsr		@r0
	mov		#IPI_CHECK_INTR, r5

	#
	# Restore context from act->reg
	#
0:	mov.l	Lid_REG_OFF, r11
	add		r12, r11						! &act->reg

	#
	# Since we are not returning via __ker_exit we need to check if the
	# kernel/ISR touched the FPU. If so, we need to disable the FPU so
	# that we force the user thread to restore its FPU context
	#
	mov.l	Lid_actives_fpu, r1
	SMP_GETCPU	r3
	SMP_ADD	r1, r3
	mov.l	@r1, r1						! actives_fpu[RUNCPU]
	cmp/eq	r1, r12
	bt		0f
	mov.l	Lid_SR_FD, r2
	mov		#REG_SR, r0
	mov.l	@(r0, r11), r1				! saved SR
	or		r2, r1						! disable FPU 
	mov.l	r1, @(r0, r11)

0:	bra		__ker_exit2
	nop
.endif

from_kerorintr:
	#
	# Interrupted kernel code
	#

.ifdef	VARIANT_smp
	#
	# Return immediately if nested interrupt or kernel is locked
	#
	# r0 has the current inkernel value from above
	#
	tst		#INKERNEL_LOCK, r0
	bf		intr_done_iret

	#
	# Must use cpupageptr[]->state to check nested interrupts on this cpu
	#
	mov.l	Lid_cpupageptr, r0
	SMP_GETCPU	r3
	SMP_ADD	r0, r3
	mov.l	@r0, r0						! cpupageptr[RUNCPU]
	mov.l	@(CPUPAGE_STATE, r0), r0	! cpupageptr[RUNCPU]->state
	tst		r0, r0
	bf		intr_done_iret

	#
	# Check if we were spinning waiting to acquire the kernel:
	#   beg_acquire_kernel <= pc <= end_acquire_kernel
	#
	mov.l	Lid_REG_PC, r0
	mov.l	Lid_beg_acquire_kernel, r1
	mov.l	@(r0, r11), r0					! interrupted pc value
	mov.l	Lid_end_acquire_kernel, r2
	cmp/hs	r1, r0
	bf		case3
	cmp/hi	r2, r0
	bf		case6
.else
	#
	# Return immediately if nested interrupt or kernel is locked
	#
	# r0 has the current inkernel value from above
	#
	tst		#INKERNEL_LOCK|INKERNEL_INTRMASK, r0
	bf		intr_done_iret
.endif

case3:
	#
	# Preempt running thread if queued_event_priority > act->priority
	#
	mov.l	Lid_PRIORITY, r0
	mov.b	@(r0, r12), r0
	mov.l	Lid_queued_event_priority, r1
	mov.l	@r1, r2
	and		#0xff,r0
	cmp/hi	r0, r2
	bt		preempt

.ifdef	VARIANT_smp
case4:
case5:
	#
	# If intrevent_pending is set, send an IPI to another cpu to handle
	# the pending events since we can't drain them here.
	#
	mov.l	Lid_intrevent_pending, r0
	mov.l	@r0, r0
	tst		r0, r0
	bt		intr_done_iret

	#
	# Send IPI to (RUNCPU + 1) % num_processors
	#
	mov.l	Lid_num_processors, r0
	stc		r6_bank, r4					! RUNCPU << 2
	shlr2	r4							! RUNCPU
	mov.l	@r0, r0
	add		#1, r4
	cmp/eq	r4, r0
	bf		0f
	mov		#0, r4
0:	mov.l	Lid_send_ipi, r0
	mov		#IPI_CHECK_INTR, r5
	jsr		@r0
	nop
.endif

intr_done_iret:
	bra		__ker_iret
	nop

.ifdef	VARIANT_smp
case6:
	#
	# If we were spinning in system mode, return immediately
	#
	mov.l	Lid_sys_acquire_kernel, r1
	cmp/hi	r1, r0
	bt		intr_done_iret

	#
	# Check if we were in __ker_entry by checking the flags being set.
	# The syscall entry is only place where we set just INKERNEL_NOW.
	#
	# For a syscall, back up the PC to restart the call.
	# Otherwise, just restore the context to regenerate the exception.
	#
	# WARNING: the register used must match usr_acquire_kernel
	#
	mov		#0, r0
	or		#INKERNEL_NOW, r0
	mov		r0, r1
	mov		#REG_R5, r0
	mov.l	@(r0, r11), r0						! interrupted r5 value
	cmp/eq	r1, r0
	bf		0f

	#
	# Back up the saved PC to restart the system call
	#
	mov.l	Lid_REG_OFF_PC, r0
	mov.l	@(r0, r12), r1						! saved user PC
	add		#-KER_ENTRY_SIZE, r1
	mov.l	r1, @(r0, r12)

0:	stc		r5_bank, r15						! reset kernel stack
	bra		case1
	nop
.endif

	#
	# Preempt the running thread:
	# - set INKERNEL_NOW|LOCK, clear INKERNEL_SPECRET
	# - back up pc to restart system call if necessary
	# - call xfer_handlers restart handler if necessary
	# - return through __ker_exit to restore new active thread
	#
preempt:
.ifdef	VARIANT_smp
	#
	# FIXME_MSGOPT: x86 has SMP_MSGOPT support here
	#

	stc		r7_bank, r1
	mov		#INKERNEL_SPECRET, r2
	not		r2, r2
0:	.word	0x0163								! movli.l @r1, r0
	and		r2, r0								! clear SPECRET
	or		#INKERNEL_NOW|INKERNEL_LOCK, r0
	.word	0x0173								! movco.l r0, @r1
	bf		0b
.else
	stc		r7_bank, r0
	or		#INKERNEL_NOW+INKERNEL_LOCK, r0
	mov		#INKERNEL_SPECRET, r1
	not		r1, r1
	and		r1, r0
	ldc		r0, r7_bank
.endif

	#
	# Restart system call if we haven't started __ker_exit processing
	#
	tst		#INKERNEL_EXIT, r0
	bf		1f
	mov.l	Lid_REG_OFF_PC, r2
	add		r12, r2
	mov.l	@r2, r1
	add		#-KER_ENTRY_SIZE, r1
	mov.l	r1, @r2
1:

	#
	# Check xfer_handlers and call the restart handler if necessary
	#
	mov.l	Lid_xfer_handlers, r1
.ifdef	SMP_MSGOPT
	SMP_GETCPU	r3
	SMP_ADD	r1, r3						! &xfer_handlers[RUNCPU]
.endif
	mov.l	@r1, r0
	cmp/eq	#0, r0
	bt		1f
	mov		#0, r2
	mov.l	r2, @r1						! clear xfer_handlers[RUNCPU]
	mov.l	@(4, r0), r0				! restart handler
	cmp/eq	#0, r0
	bt		1f

	#
	# Call restart_handler(act, reg)
	#
	mov		r12, r4
	jsr		@r0
	mov		r11, r5

1:	bra		__ker_exit
	nop

routine_end intr_done

	.align 2

Lid_interrupt:				.long	interrupt
Lid_REG_OFF:				.long	REG_OFF
Lid_REG_OFF_PC:				.long	REG_OFF+REG_PC
Lid_xfer_handlers:			.long	xfer_handlers
Lid_ker_exit:				.long	__ker_exit
Lid_PRIORITY:				.long	PRIORITY
Lid_queued_event_priority:	.long	queued_event_priority
.ifdef	VARIANT_smp
Lid_intr_slock:				.long	intr_slock
Lid_send_ipi:				.long	send_ipi
Lid_cpupageptr:				.long	cpupageptr
Lid_REG_PC:					.long	REG_PC
Lid_beg_acquire_kernel:		.long	beg_acquire_kernel
Lid_end_acquire_kernel:		.long	end_acquire_kernel
Lid_sys_acquire_kernel:		.long	sys_acquire_kernel
Lid_intrevent_pending:		.long	intrevent_pending
Lid_num_processors:			.long	num_processors
Lid_actives_fpu:			.long	actives_fpu
Lid_SR_FD:					.long	SH_SR_FD
.endif

.ifdef	VARIANT_smp
#-------------------------------------------------------------------------
#
# intr_process_ipi: handle inter-processor interrupts
#
# On entry:
# - interrupts are disabled
# - r11 = saved context
# - r13 = 1 if entered from kernel, 0 if entered from user
#
#-------------------------------------------------------------------------

routine_start intr_process_ipi, 1
	#
	# Save callee-saved registers we modify
	#
	mov.l	r8,  @-r15
	mov.l	r9,  @-r15
	mov.l	r10, @-r15
	mov.l	r14, @-r15
	sts.l	pr,  @-r15

	RELEASE_INTR_SLOCK	Lip_intr_slock
	#
	# Set cpupageptr[RUNCPU]->state = 1 to indicate we're in an ISR
	#
	mov.l	Lip_cpupageptr, r0
	SMP_GETCPU	r3
	SMP_ADD	r0, r3
	mov.l	@r0, r0						! cpupageptr[RUNCPU]
	mov		#1, r1
	mov.l	@(CPUPAGE_STATE, r0), r8	! remember previous state
	mov.l	r1, @(CPUPAGE_STATE, r0)

	#
	# Set __shadow_imask and remember current value to restore on exit
	#
	mov.l	Lip_cpu_imask, r1
	stc		sr, r0
	SMP_ADD	r1, r3
	mov.l	@r1, r1						! __cpu_imask[RUNCPU]
	and		#SH_SR_IMASK, r0
	mov.l	@r1, r14					! r14 = *__cpu_imask[RUNCPU]
	mov.l	r0, @r1						! *__cpu_imask[RUNCPU] = SR.IMASK

	#
	# cmds = _smp_xchg(&ipicmds[RUNCPU], 0)
	#
	mov.l	Lip_ipicmds, r1
	SMP_ADD	r1, r3
0:	.word	0x0163						! movli.l @r1, r0
	mov		r0, r9						! r9 = pending ipi cmds
	mov		#0, r0
	.word	0x0173						! movco.l r0, @r1
	bf		0b

	mov		r9, r0
	tst		#IPI_PARKIT, r0
	bt		1f

	! Freeze the system, we've got a kernel dump happening
	mov.l	Lalives,r0
	shlr2	r3
	mov.b	@(r0,r3),r2
	mov		#2,r4
	or		r4,r2
	mov.b	r2,@(r0,r3)	! mark the CPU as parked
2:	bra		2b	

	
1:	
	tst		#IPI_TLB_SAFE, r0
	bt		1f
	#
	# Call set_safe_aspace()
	#
	mov.l	Lip_set_safe_aspace, r0
	stc		r6_bank, r4
	jsr		@r0
	shlr2	r4							! RUNCPU
	nop
	#
	# set_safe_aspace() may cause interrupts to be re-enabled
	#
	stc		sr, r0
	or		#SH_SR_IMASK, r0
	ldc		r0, sr

1:	mov.l	Lip_IPI_CLOCK_LOAD, r0
	tst		r0, r9
	bt		1f
	#
	# Call clock_load()
	#
	mov.l	Lip_clock_load, r0
	jsr		@r0
	nop
	#
	# clock_load() may cause interrupts to be re-enabled
	#
	stc		sr, r0
	or		#SH_SR_IMASK, r0
	ldc		r0, sr

1:	mov.l	Lip_IPI_INTR_MASK, r0
	tst		r0, r9
	bt		1f
	#
	# Call interrupt_smp_sync(INTR_FLAG_SMP_BROADCAST_MASK)
	#
	mov.l	Lip_interrupt_smp_sync, r0
	mov.l	Lip_INTR_FLAG_SMP_BROADCAST_MASK, r4
	jsr		@r0
	nop

1:	mov.l	Lip_IPI_INTR_UNMASK, r0
	tst		r0, r9
	bt		1f
	#
	# Call interrupt_smp_sync(INTR_FLAG_SMP_BROADCAST_MASK)
	#
	mov.l	Lip_interrupt_smp_sync, r0
	mov.l	Lip_INTR_FLAG_SMP_BROADCAST_UNMASK, r4
	jsr		@r0
	nop

1:	mov		r9, r0
	tst		#IPI_TLB_FLUSH, r0
	bt		1f
	#
	# Flush TLB: smp_sync_tlb(aspaces_prp[RUNCPU])
	#
	# FIXME: this makes a direct call to a memmgr function.
	#        This currently prevents creation of a standalone kernel.
	mov.l	Lip_smp_tlb_sync, r0
	mov.l	Lip_aspaces_prp, r4
	SMP_GETCPU	r1
	SMP_ADD	r4, r1						! &aspaces_prp[RUNCPU]
	jsr		@r0
	mov.l	@r4, r4						! aspaces_prp[RUNCPU]
	#
	# smp_sync_tlb() may cause interrupts to be re-enabled
	#
	stc		sr, r0
	or		#SH_SR_IMASK, r0
	ldc		r0, sr

1:	mov		#0, r10						! to accumulate async_flags settings
	mov		r9, r0
	tst		#IPI_RESCHED, r0
	bt		1f
	mov		#_NTO_ATF_SMP_RESCHED, r0
	or		r0, r10

1:	mov		r9, r0
	tst		#IPI_TIMESLICE, r0
	bt		1f
	mov		#_NTO_ATF_TIMESLICE, r0
	or		r0, r10

1:	mov		r9, r0
	tst		#IPI_CONTEXT_SAVE, r0
	bt		1f
	#
	# Call cpu_force_fpu_save(actives_fpu[RUNCPU])
	#
	mov.l	Lip_actives_fpu, r0
	mov.l	Lip_cpu_force_fpu_save, r1
	SMP_GETCPU	r3
	SMP_ADD	r0, r3
	mov.l	@r0, r4
	tst		r4, r4
	bt		1f
	jsr		@r1
	nop

	#
	# Clear actives_fpu[RUNCPU]
	#
	mov.l	Lip_actives_fpu, r0
	SMP_GETCPU	r3
	SMP_ADD	r0, r3
	mov		#0, r1
	mov.l	r1, @r0

	#
	# Check if we need to set actives[RUNCPU]->async_flags
	# r10 contains any pending async_flags set above
	#
	# If we entered from user mode or were spinning waiting to acquire the
	# kernel we must force a __KER_NOP system call to process async_flags:
	#
	#	if (pending_async_flags)  {
	#		if (user) {
	#			pending_async_flags |= _NTO_ATF_FORCED_KERNEL
	#		} else {
	#			if (pc >= usr_acquire_kernel && pc < sys_acquire_kernel) {
	#				if (trying to set only INKERNEL_NOW) {
	#					act->reg.gr[PC] -= KER_ENTRY_SIZE;
	#					pending_async_flags |= _NTO_ATF_FORCED_KERNEL
	#					r13 = 0 (pretend we entered from user)
	#				}
	#			}
	#		}
	#		atflags = _smp_xchg(&act->async_flags, pending_async_flags)
	#		if ((pending_async_flags & _NTO_ATF_FORCED_KERNEL) &&
	#			!(atflags & _NTO_ATF_FORCED_KERNEL)) {
	#			act->args.async.save_ip = act->reg.gp[PC]
	#			act->reg.gp = _kercallptr + 4
	#		}
	#	}
	#
1:	tst		r10, r10
	bt		intr_process_ipi_exit

	#
	# WARNING: reuse r9 as we don't need to check any more ipicmds
	#
	stc		r4_bank, r9						! &actives[RUNCPU]
	mov.l	@r9, r9							! actives[RUNCPU]

	#
	# If we entered from user mode, set _NTO_ATF_FORCED_KERNEL
	#
	tst		r13, r13
	bt		2f
	#
	# Check if we were spinning waiting to acquire the kernel
	#
	mov		#REG_PC, r0
	mov.l	Lip_usr_acquire_kernel, r1
	mov.l	@(r0, r11), r0					! interrupted pc value
	mov.l	Lip_sys_acquire_kernel, r2
	cmp/hs	r1, r0
	bf		3f								! pc < usr_acquire_kernel
	cmp/hs	r2, r0
	bt		3f								! pc >= sys_acquire_kernel
	#
	# Spinning in usr_acquire_kernel - check if we were setting INKERNEL_NOW
	# WARNING: we assume the flags are in r5.
	#
	mov		#0, r0
	or		#INKERNEL_NOW, r0
	mov		r0, r1
	mov		#REG_R5, r0
	mov.l	@(r0, r11), r0
	cmp/eq	r1, r0
	bf		3f

	#
	# Back up saved PC to restart the system call
	# Set r13 to pretend we entered from user mode
	#
	mov.l	Lip_REG_OFF_PC, r0
	mov.l	@(r0, r9), r2					! user pc value
	mov		#0, r13
	add		#-KER_ENTRY_SIZE, r2
	mov.l	r2, @(r0, r9)

2:	mov.l	Lip_ATF_FORCED_KERNEL, r1
	or		r1, r10

3:	#
	# Set act->async_flags
	#
	mov		r9, r1
	add		#ATFLAGS, r1
0:	.word	0x0163							! movli.l @r1, r0
	mov		r0, r2
	or		r10, r0							! set async_flags
	.word	0x0173							! movco.l r0, @r1
	bf		0b

	#
	# Only force call if _NTO_ATF_FORCED_KERNEL was not already set
	#
	mov.l	Lip_ATF_FORCED_KERNEL, r1
	tst		r10, r1
	bt		intr_process_ipi_exit
	tst		r2, r1
	bf		intr_process_ipi_exit

	#
	# Save user pc in act->args.async.save_ip
	# Set user pc to execute __KER_NOP call at __kercallptr+4
	#
	mov.l	Lip_REG_OFF_PC, r0
	mov.l	Lip_ARGS_ASYNC_IP, r1
	mov.l	@(r0, r9), r2					! user pc
	add		r9, r1
	mov.l	r2, @r1							! act->args.async.save_ip = user pc
	mov.l	Lip_kercallptr, r1
	mov.l	@r1, r1
	add		#4, r1							! address of __KER_NOP call
	mov.l	r1, @(r0, r9)					! set pc to __KER_NOP call

intr_process_ipi_exit:
	#
	# Restore cpupageptr[RUNCPU]->state and return into generated eoi code
	#
	mov.l	Lip_cpupageptr, r0
	SMP_GETCPU	r3
	SMP_ADD	r0, r3
	mov.l	@r0, r0							! cpupageptr[RUNCPU]
	mov.l	r8, @(CPUPAGE_STATE, r0)

	#
	# Restore shadow_imask
	#
	mov.l	Lip_cpu_imask, r0
	SMP_ADD	r0, r3
	mov.l	@r0, r0							! __cpu_imask[RUNCPU]
	mov.l	r14, @r0						! *__cpu_imask[RUNCPU] = old value

	ACQUIRE_INTR_SLOCK	Lip_intr_slock

	lds.l	@r15+, pr
	mov.l	@r15+, r14
	mov.l	@r15+, r10
	mov.l	@r15+, r9
	mov.l	@r15+, r8
	rts
	nop
	
	.align 2

Lip_intr_slock:						.long	intr_slock
Lip_cpu_imask:						.long	__cpu_imask
Lip_cpupageptr:						.long	cpupageptr
Lip_ipicmds:						.long	ipicmds
Lip_IPI_CLOCK_LOAD:					.long	IPI_CLOCK_LOAD
Lip_IPI_INTR_MASK:					.long	IPI_INTR_MASK
Lip_IPI_INTR_UNMASK:				.long	IPI_INTR_UNMASK
Lip_set_safe_aspace:				.long	set_safe_aspace
Lip_smp_tlb_sync:					.long	smp_tlb_sync
Lip_aspaces_prp:					.long	aspaces_prp
Lip_clock_load:						.long	clock_load
Lip_interrupt_smp_sync:				.long	interrupt_smp_sync
Lip_INTR_FLAG_SMP_BROADCAST_MASK:	.long	INTR_FLAG_SMP_BROADCAST_MASK
Lip_INTR_FLAG_SMP_BROADCAST_UNMASK:	.long	INTR_FLAG_SMP_BROADCAST_UNMASK
Lip_actives_fpu:					.long	actives_fpu
Lip_cpu_force_fpu_save:				.long	cpu_force_fpu_save
Lip_ATF_FORCED_KERNEL:				.long	_NTO_ATF_FORCED_KERNEL
Lip_ARGS_ASYNC_IP:					.long	ARGS_ASYNC_IP
Lip_REG_OFF_PC:						.long	REG_OFF+REG_PC
Lip_usr_acquire_kernel:				.long	usr_acquire_kernel
Lip_sys_acquire_kernel:				.long	sys_acquire_kernel
Lip_kercallptr:						.long	kercallptr
Lalives:							.long	alives

routine_end intr_process_ipi
.endif

#-------------------------------------------------------------------------
#
# General exception handling.
#
# The code between __exc_general_start and __exc_general_end is copied to
# VBR+SH_EXC_GENERAL by init_traps().
#
# We are executing in bank 1 (SR.BL=1, SR.RB=1):
#
# r4_bank1 = &actives[RUNCPU]
# r5_bank1 = ker_stack[RUNCPU]
# r6_bank1 = RUNCPU << 2
# r7_bank1 = inkernel (non-SMP), &inkernel (SMP)
#
#-------------------------------------------------------------------------

	.align 5

__exc_general_start:
	#
	# Get EXPEVT value into r3 and check for special cases
	#
	mov.l	Leg_expevt, r1
	mov.l	Leg_SH_EXC_CODE_TRAP, r2
	mov.l	@r1, r3

	#
	# Check for kernel call (trapa instruction)
	#
	mov.l	Leg_ker_entry, r1
	cmp/eq	r3, r2
	bf		0f
	jmp		@r1
	nop
0:
	#
	# Check for emulation for __inline_InterruptEnable()/Disable() code.
	#
	mov.l	Leg_SH_EXC_CODE_GILLINST, r2
	mov.l	Leg_SH_EXC_CODE_SILLINST, r1
	cmp/eq	r3, r2
	bt		__exc_emu_interrupt_mask
	cmp/eq	r3, r1
	bf		__exc_general_handler

	#
	# Emulation for interrupt enable/disable code in user mode
	#
__exc_emu_interrupt_mask:
	#
	# Don't bother with emulation if (act->flags & _NTO_TF_IOPRIV) == 0)
	#
	mov.l	Leg_TFLAGS, r0
	mov.l	@r4, r2
	add		r2, r0						! &act->flags
	mov.l	Leg_TF_IOPRIV, r2
	mov.l	@r0, r0
	tst		r0, r2
	bt		__exc_general_handler

	#
	# Check if the faulting instruction sequence matches the code for
	# __inline_InterruptEnable()/__inline_InterruptDisable().
	#
	# TLB miss exceptions could occur accessing the instruction sequence,
	# so we need to save sufficient registers to recover from the TLB miss.
	#
	mov.l	Leg_ker_emu_save, r2
	SMP_GETCPU_BANK1 r0
	SMP_ADD	r2, r0
	mov.l	@r2, r2						! __ker_emu_save[RUNCPU]
	add		#20, r2
	stc.l	r0_bank, @-r2
	stc.l	r1_bank, @-r2
	stc.l	ssr, @-r2
	stc.l	spc, @-r2
	mov.l	r3, @-r2

	#
	# Switch to Bank0 with interrupts disabled
	#
	mov.l	Leg_SR_not_BL_RB, r1
	stc		sr, r0
	and		r1, r0
	or	#SH_SR_IMASK, r0
	ldc		r0, sr

	#
	# Prefetch the instruction addresses to ensure TLBs are present
	#
	stc		spc, r1
	ocbp	@r1
	add		#4, r1
	ocbp	@r1
	add		#-4, r1
	pref	@r1
	add		#4, r1
	pref	@r1

	#
	# Switch back to Bank1 and reload the saved registers
	#
	mov.l	Leg_SR_RB, r1
	stc		sr, r0
	or		r1, r0
	ldc		r0, sr

	mov.l	Leg_ker_emu_save, r2
	SMP_GETCPU_BANK1	r0
	SMP_ADD	r2, r0
	mov.l	@r2, r2						! __ker_emu_save[RUNCPU]
	mov		#0, r0
	mov.l	@r2, r3						! save EXPEVT code
	mov.l	r0, @r2						! clear active indicator
	add		#4, r2
	mov.l	@r2+, r1					! saved pc
	ldc		r1,spc
	ldc.l	@r2+,ssr
	ldc.l	@r2+, r1_bank
	ldc.l	@r2+, r0_bank

	#
	# Check the faulting instruction sequence
	#
	# This code assumes the __inline_InterruptEnable()/Disable() code is:
	#
	# __inline_Interrupt_Enable:
	# 	stc		sr, r0
	#	and		rm, r0		where rm = *__shadow_imask | ~SH_SR_IMASK
	#	ldc		r0, sr
	#
	# __inline_Interrupt_Disable:
	# 	stc		sr, r0
	#	or		0xf0, r0
	#	ldc		r0, sr
	#
	mov.w	@r1, r0
	mov.w	Leg_opcode_stc_sr_r0, r2	! is 1st opcode stc sr, r0?
	cmp/eq	r0, r2
	bf		__exc_general_handler

	mov.w	Leg_opcode_ldc_r0_sr, r2	! is 3rd opcode ldc r0, sr?
	mov.w	@(4, r1), r0
	cmp/eq	r0, r2
	bf		__exc_general_handler

	#
	# Check if it's __inline_InterruptEnable or __inline_InterruptDisable
	#
	mov.w	Leg_opcode_or_0xf0_r0, r2
	mov.w	@(2, r1), r0				! is 2nd opcode or 0xf0, r0?
	cmp/eq	r0, r2
	bf		__exc_emu_interrupt_enable

	#
	# Skip past the ldc r0, sr instruction
	#
	add		#6, r1
	ldc		r1,spc

	#
	# Set the SR.IMASK and return from exception
	#
	stc		ssr, r0
	or		#SH_SR_IMASK, r0
	ldc		r0,ssr
	rte
	nop

__exc_emu_interrupt_enable:
	mov.w	Leg_opcode_and_rm_r0_mask, r2
	mov.w	Leg_opcode_and_rm_r0, r1
	and		r0, r2						! is 2nd opcode and rm, r0?
	cmp/eq	r2, r1
	mov.w	Leg_opcode_and_rm_field_mask, r2
	bf		__exc_general_handler

	#
	# Extract rm field and use jump table to pick up the register value
	#
	and		r0, r2

	.align 2
	mova	@(12,pc), r0
	shlr2	r2
	mov.w	Leg_SR_not_IMASK, r1
	add		r2, r0
	jmp		@r0
	nop

	bra		0f
	stc		r0_bank, r2
	bra		0f
	stc		r1_bank, r2
	bra		0f
	stc		r2_bank, r2
	bra		0f
	stc		r3_bank, r2
	bra		0f
	stc		r4_bank, r2
	bra		0f
	stc		r5_bank, r2
	bra		0f
	stc		r6_bank, r2
	bra		0f
	stc		r7_bank, r2
	bra		0f
	mov		r8, r2
	bra		0f
	mov		r9, r2
	bra		0f
	mov		r10, r2
	bra		0f
	mov		r11, r2
	bra		0f
	mov		r12, r2
	bra		0f
	mov		r13, r2
	bra		0f
	mov		r14, r2
	mov		r15, r2
0:
	#
	# Only update the SR.IMASK field if rm is ~SH_SR_IMASK
	# FIXME: why? What if rm contained a non-zero __shadow_imask value?
	#
	cmp/eq	r2, r1
	bf		__exc_general_handler

	#
	# Set new SR value with imask field set as per __shadow_imask
	#
	mov.l	Leg_shadow_imask, r2
	mov.l	Leg_SR_IMASK, r3
	mov.l	@r2, r1
	mov.l	Leg_SR_not_IMASK, r2
	mov.l	@r1, r0
	stc		ssr, r1
	and		r3, r0
	and		r2, r1
	or		r0, r1
	ldc		r1,ssr

	#
	# Skip past the ldc r0, sr instruction and return from exception
	#
	stc		spc, r3
	add		#6, r3
	ldc		r3,spc
	rte
	nop

	#
	# General exception handler:
	# - save context and acquire kernel
	# - branch to appropriate handler function
	#
	# We are executing in bank1 (SR.BL=1, SR.RB=1)
	#
	# r3_bank1 contains the EXPEVT code
	#
__exc_general_handler:
	#
	# Save context
	#
	EXC_SAVECONTEXT

	#
	# Convert EXPEVT to jump table index and invoke the appropriate handler
	#
	shlr2	r3
	mov		#0x3c, r1
	shlr	r3
	mova	__exc_general_exptable, r0
	and		r3, r1
	mov.l	@(r0, r1), r0
	jmp		@r0
	nop

	.align 2

Leg_expevt:						.long	SH_MMR_CCN_EXPEVT
Leg_SH_EXC_CODE_TRAP:			.long	SH_EXC_CODE_TRAP
Leg_ker_entry:					.long	__ker_entry
Leg_ker_emu_save:				.long	__ker_emu_save
Leg_shadow_imask:				.long	__shadow_imask
Leg_SH_EXC_CODE_GILLINST:		.long	SH_EXC_CODE_GILLINST
Leg_SH_EXC_CODE_SILLINST:		.long	SH_EXC_CODE_SILLINST
Leg_TFLAGS:						.long	TFLAGS
Leg_TF_IOPRIV:					.long	_NTO_TF_IOPRIV
Leg_SR_not_BL_RB:				.long	~(SH_SR_BL | SH_SR_RB)
Leg_SR_IMASK:					.long	SH_SR_IMASK
Leg_SR_RB:						.long	SH_SR_RB
Leg_SR_not_IMASK:				.long	~SH_SR_IMASK
Leg_opcode_stc_sr_r0:			.word	0x2
Leg_opcode_ldc_r0_sr:			.word	0x400e
Leg_opcode_or_0xf0_r0:			.word	0xcbf0
Leg_opcode_and_rm_r0_mask:		.word	0xff0f
Leg_opcode_and_rm_r0:			.word	0x2009
Leg_opcode_and_rm_field_mask:	.word	0xf0

	.align 2

__exc_general_exptable:
	.long	__exc_fpu_disabled		/* 0x800 */
	.long	__exc_fpu_disabled_slot	/* 0x820 */
	.long	__exc_tlb_miss_read		/* 0x040 */
	.long	__exc_tlb_miss_write	/* 0x060 */
	.long	__exc_init_page_write	/* 0x080 */
	.long	__exc_tlb_prot_read		/* 0x0a0 */
	.long	__exc_tlb_prot_write	/* 0x0c0 */
	.long	__exc_access_read		/* 0x0e0 */
	.long	__exc_access_write		/* 0x100 */
	.long	__exc_fpu				/* 0x120 */
	.long	__exc_unexpected		/* 0x140 */
	.long	__exc_unexpected		/* 0x160 - special case code above   */
	.long	__exc_ill_instr			/* 0x180 */
	.long	__exc_ill_instr_slot	/* 0x1a0 */
	.long	__exc_unexpected		/* 0x1c0 */
	.long	__exc_usr_break			/* 0x1e0 */

__exc_general_end:
	 .type __exc_general_start, @function
	 .type __exc_general_end, @function

#-------------------------------------------------------------------------
# End of copied exception code
#-------------------------------------------------------------------------


#-------------------------------------------------------------------------
# FPU exception
#
# On entry:
# - executing in bank 1 (SR.BL=1, SR.RB=1):
# - r11 = saved context
# - r12 = actives[RUNCPU]
# - r14 = kernel/user mode indicator: 1 - kernel, 0 - user
#-------------------------------------------------------------------------

__exc_fpu:
	#
	# Acquire kernel and switch to bank0 with interrupts and FPU disabled
	#
	EXC_ENTERKERNEL

	mov.l	Lef_SIGFPE_code, r4
	bra 	__exc
	nop

	.align 2

Lef_SIGFPE_code:		.long	SIGFPE+ (FPE_FLTDIV*256) + (FLTFPE*65536)

#-------------------------------------------------------------------------
# FPU disable exception
#
# Note that kernel/interrupt code can use the FPU because the compiler uses
# FPU operations for integer divide.
#
# On entry:
# - executing in bank0 with interrupts disabled
# - r11 = saved context
# - r12 = actives[RUNCPU]
# - r14 = kernel/user mode indicator: 1 - kernel, 0 - user
#-------------------------------------------------------------------------

__exc_fpu_disabled:
__exc_fpu_disabled_slot:
	#
	# Acquire kernel and switch to bank0 with interrupts and FPU disabled
	#
	EXC_ENTERKERNEL

	#
	# Enable FPU
	#
	mov.l	Lef_SR_not_FD, r1
	stc		sr, r0
	and		r1, r0
	ldc		r0, sr

	#
	# Save current active FPU context if actives_fpu[RUNCPU] is non-null
	#
	mov.l	Lef_actives_fpu, r10
	SMP_GETCPU	r0
	SMP_ADD	r10, r0						! actives_fpu[RUNCPU]
	mov.l	@r10, r4
	tst		r4, r4
	bt		0f
	bsr		cpu_force_fpu_save
	nop
	mov		#0, r0
	mov.l	r0, @r10					! set actives_fpu[RUNCPU] = 0
0:

	#
	# Are we coming from user-mode or from kernel call/interrupt/exception?
	#
	tst		r14, r14
	bt		__exc_fpu_restore	! if user-mode, branch to __exc_fpu_restore

	#
	# We are coming from kernel mode.  
	#
	# Check ker_savefpu to see if we need to save the kernel FPU context.
	#
	mov.l	Lef_ker_savefpu,r1	! r1 = &ker_savefpu[0]
	SMP_GETCPU	r0				! r0 = RUNCPU<<2
	SMP_ADD	r1, r0				! r1 = &ker_savefpu[RUNCPU]
	mov.l	@r1,r1				! r1 = ker_savefpu[RUNCPU]
	tst		r1,r1
	bt		Lef_ker_done		! if ker_savefpu==NULL, branch to Lef_ker_done
	
	# ker_savefpu is non-zero, so somebody is expecting to have their
	# fpu context backed up.  Check ker_savefpu->fpscr to see if the
	# context has already been saved (no point doing it twice).
	mov.l	Lef_FPSCR,r0	! offset of FPSCR in fpu context
	mov.l	@(r0,r1),r0		! r0 = ker_savefpu->fpscr
	tst		r0,r0
	bf		Lef_ker_done	! if ker_savefpu->fpscr!=0, branch to Lef_ker_done

	# Need to save context.
	SAVE_FPU_REGISTERS	r1
	
Lef_ker_done:
	#
	# Set correct initial fpscr bits: double precision, bank 0
	#
	mov.l	Lef_FPSCR_DN_PR, r0
	lds		r0, fpscr

	#
	# Enable FPU in saved SR register and restore context
	#
	mov		#REG_SR, r0
	mov.l	Lef_SR_not_FD, r2
	mov.l	@(r0, r11), r1
	and		r2, r1
	mov.l	r1, @(r0, r11)

.ifdef	VARIANT_smp
	#
	# Restore only the inkernel bits - other cpus can change the intr count
	#
	mov.l	Lef_INKERNEL_bits, r2
	stc		r7_bank, r1
	and		r2, r13						! old inkernel bits
	not		r2, r2
0:	.word	0x0163						! movli.l @r1, r0
	and		r2, r0						! clear inkernel bits
	or		r13, r0						! set old inkernel bits
	.word	0x0173						! movco.l r0, @r1
	bf		0b
	bra		__ker_iret
	nop
.else
	bra		__ker_iret
	ldc		r13, r7_bank
.endif

__exc_fpu_restore:
	#
	# Restore FPU context from act->fpudata.
	#
	# If act->fpudata == 0, need to allocate FPU save area
	#
	mov.l	Lef_FPUDATA, r8
	add		r12, r8
	mov.l	@r8, r9
	tst		r9, r9
	bt		__exc_fpu_save_alloc

.ifdef	VARIANT_smp
	#
	# If context was active on another CPU we need to send an IPI to force
	# it to be saved so we can restore the context on this CPU:
	#
	#	if (FPUDATA_BUSY(fpudata) && FPUDATA_CPU(fpudata) != RUNCPU) {
	#		SENDIPI(FPUDATA_CPU(fpudata), IPI_CONTEXT_SAVE)
	#		bitset_inkernel(INKERNEL_EXIT);
	#		unlock_kernel();
	#		InterruptEnable();
	#		while (FPUDATA_BUSY(fpudata))
	#			;
	#		InterruptDisable();
	#		lock_kernel();
	#	}
	#	ptr = FPUDATA_PTR(fpudata);
	#
	mov		r9, r0
	tst		#FPUDATA_BUSY, r0
	bt		__exc_fpu_not_busy
	SMP_GETCPU	r1						! RUNCPU << 2
	shlr2	r1							! RUNCPU
	and		#FPUDATA_CPUMASK, r0		! FPUDATA_CPU(fpudata)
	cmp/eq	r0, r1
	bt		__exc_fpu_not_busy

	#
	# send_ipi(cpu, IPI_CONTEXT_SAVE)
	#
	mov.l	Lef_send_ipi, r1
	mov		r0, r4
	jsr		@r1
	mov		#IPI_CONTEXT_SAVE, r5

	#
	# Unlock the kernel and enable interrupts while we spin.
	# Set INKERNEL_EXIT so we restart faulting instruction if we are preempted.
	#
	stc		r7_bank, r1					! &inkernel
	mov.l	Lef_not_INKERNEL_LOCK, r2
0:	.word	0x0163						! movli.l @r1, r0
	or		#INKERNEL_NOW|INKERNEL_EXIT, r0
	and		r2, r0
	.word	0x0173						! movco.l r0, @r1
	bf		0b

	# FIXME: should use __shadow_imask?
	mov.l	Lef_SR_not_IMASK, r2
	stc		sr, r0
	and		r2, r0
	ldc		r0, sr

	#
	# Wait for FPUDATA_BUSY to be cleared by IPI handler on other CPU
	#
0:	mov.l	@r8, r0
	tst		#FPUDATA_BUSY, r0
	bf		0b
	mov		r0, r9

	#
	# Disable interrupts and relock the kernel
	#
	stc		sr, r0
	or		#SH_SR_IMASK, r0
	ldc		r0, sr
0:	.word	0x0163						! movli.l @r1, r0
	or		#INKERNEL_LOCK, r0
	.word	0x0173						! movco.l r0, @r1
	bf		0b

__exc_fpu_not_busy:
	#
	# FIXME: this should not be necessary - bits should be 0 if not busy
	#
	mov.l	Lef_FPUDATA_MASK, r1
	and		r1, r9						! FPUDATA_PTR(fpudata)
.endif

	#
	# Set actives_fpu[RUNCPU] = act and restore FPU context
	#
	mov.l	r12, @r10

	RESTORE_FPU_REGISTERS r9

.ifdef	VARIANT_smp
	#
	# Indicate context is active on this cpu:
	#	act->fpudata |= FPUDATA_BUSY | RUNCPU
	#
	mov.l	Lef_SH_FPU_REG_SIZE, r1
	mov		r9, r0
	sub		r1, r0
	SMP_GETCPU	r1						! RUNCPU << 2
	shlr2	r1							! RUNCPU
	or		r1, r0
	or		#FPUDATA_BUSY, r0
	mov.l	r0, @r8
.endif

__exc_fpu_enable:
	#
	# Enable FPU in the saved SR register
	#
	mov		#REG_SR, r0
	mov.l	Lef_SR_not_FD, r2
	mov.l	@(r0, r11), r1
	and		r2, r1
	mov.l	r1, @(r0, r11)

	# restore to old inkern value
.ifdef	VARIANT_smp
	#
	# We must return via __ker_exit:
	# - actives[RUNCPU] may have changed if we had to spin above
	# - another cpu may have queued intrevents while we had the kernel locked
	#
	bra		__ker_exit
	nop
.else
	bra		__ker_iret
	ldc		r13, r7_bank
.endif

__exc_fpu_save_alloc:
	#
	# Set act->async flags to allocate save area on return via __ker_exit
	#
	mov.l	Lef_ATFLAGS, r1
	add		r12, r1
.ifdef	VARIANT_smp
	mov.l	Lef_ATF_FPUSAVE_ALLOC, r2
0:	.word	0x0163						! movli.l @r1, r0
	or		r2, r0
	.word	0x0173						! movco.l r0, @r1
	bf		0b
.else
	mov.l	@r1, r0
	mov.l	Lef_ATF_FPUSAVE_ALLOC, r2
	or		r2, r0
	mov.l	r0, @r1
.endif

	bra 	__ker_exit
	nop

	.align 2

Lef_FPSCR:				.long	REG_FPSCR
Lef_ker_savefpu:		.long	ker_savefpu
Lef_SR_not_FD:			.long	~SH_SR_FD
Lef_ATF_FPUSAVE_ALLOC:	.long	_NTO_ATF_FPUSAVE_ALLOC
Lef_ATFLAGS:			.long	ATFLAGS
Lef_FPUDATA:			.long	FPUDATA
Lef_actives_fpu:		.long	actives_fpu
Lef_FPSCR_DN_PR:		.long	SH_FPSCR_DN | SH_FPSCR_PR ! double precision, bank 0
.ifdef	VARIANT_smp
Lef_INKERNEL_bits:		.long	(INKERNEL_NOW|INKERNEL_LOCK|INKERNEL_SPECRET|INKERNEL_EXIT)
Lef_send_ipi:			.long	send_ipi
Lef_SR_not_IMASK:		.long	~SH_SR_IMASK
Lef_not_INKERNEL_LOCK:	.long	~INKERNEL_LOCK
Lef_SH_FPU_REG_SIZE:	.long	SH_FPU_REG_SIZE
Lef_FPUDATA_MASK:		.long	FPUDATA_MASK
.endif


#-------------------------------------------------------------------------
# EXPEVT 040 - data TLB miss (read) not handled by TLB miss handler
#
# On entry:
# - executing in bank 1 (SR.BL=1, SR.RB=1):
# - r11 = saved context
# - r12 = actives[RUNCPU]
# - r14 = kernel/user mode indicator: 1 - kernel, 0 - user
#-------------------------------------------------------------------------

__exc_tlb_miss_read:
	#
	# Set signal code and jump to common handler
	#
	mov.l	0f, r3
	bra		__exc_access
	nop

	.align 2

0:	.long	SIGSEGV | (FLTPAGE << 16) | (SEGV_MAPERR << 8)

#-------------------------------------------------------------------------
# EXPEVT 060 - data TLB miss (write) not handled by TLB miss handler
#
# On entry:
# - executing in bank 1 (SR.BL=1, SR.RB=1):
# - r11 = saved context
# - r12 = actives[RUNCPU]
# - r14 = kernel/user mode indicator: 1 - kernel, 0 - user
#-------------------------------------------------------------------------

__exc_tlb_miss_write:
	#
	# Set signal code and jump to common handler
	#
	mov.l	0f, r3
	bra		__exc_access
	nop

	.align 2

0:	.long	SIGSEGV | (FLTPAGE << 16) | (SEGV_MAPERR << 8) | SIGCODE_STORE

#-------------------------------------------------------------------------
# EXPEVT 080 - initial page write
#
# On entry:
# - executing in bank 1 (SR.BL=1, SR.RB=1):
# - r11 = saved context
# - r12 = actives[RUNCPU]
# - r14 = kernel/user mode indicator: 1 - kernel, 0 - user
#-------------------------------------------------------------------------

__exc_init_page_write:
	#
	# Set signal code and jump to common handler
	#
	mov.l	0f, r3
	bra		__exc_access
	nop

	.align 2

0:	.long	SIGSEGV | (FLTPAGE << 16) | (SEGV_ACCERR << 8) | SIGCODE_STORE

#-------------------------------------------------------------------------
# EXPEVT 0A0 - data TLB protection violation (read)
#            - instruction TLB protection violation
#
# On entry:
# - executing in bank 1 (SR.BL=1, SR.RB=1):
# - r11 = saved context
# - r12 = actives[RUNCPU]
# - r14 = kernel/user mode indicator: 1 - kernel, 0 - user
#-------------------------------------------------------------------------

__exc_tlb_prot_read:
	#
	# Set signal code and jump to common handler
	#
	mov.l	0f, r3
	bra		__exc_access
	nop

	.align 2

0:	.long	SIGSEGV | (FLTPAGE << 16) | (SEGV_ACCERR << 8)

#-------------------------------------------------------------------------
# EXPEVT 0C0 - data TLB protection violation (write)
#
# On entry:
# - executing in bank 1 (SR.BL=1, SR.RB=1):
# - r11 = saved context
# - r12 = actives[RUNCPU]
# - r14 = kernel/user mode indicator: 1 - kernel, 0 - user
#-------------------------------------------------------------------------

__exc_tlb_prot_write:
	#
	# Set signal code and jump to common handler
	#
	mov.l	0f, r3
	bra		__exc_access
	nop

	.align 2

0:	.long	SIGSEGV | (FLTPAGE << 16) | (SEGV_ACCERR << 8) | SIGCODE_STORE

#-------------------------------------------------------------------------
# EXPEVT 0E0 - instruction address error
#            - data address error (read)
#
# On entry:
# - executing in bank 1 (SR.BL=1, SR.RB=1):
# - r11 = saved context
# - r12 = actives[RUNCPU]
# - r14 = kernel/user mode indicator: 1 - kernel, 0 - user
#-------------------------------------------------------------------------

__exc_access_read:
	#
	# Currently fall through into __exc_access_write
	#

#-------------------------------------------------------------------------
# EXPEVT 0E0 - instruction address error
#            - data address error (read)
#
# On entry:
# - executing in bank 1 (SR.BL=1, SR.RB=1):
# - r11 = saved context
# - r12 = actives[RUNCPU]
# - r14 = kernel/user mode indicator: 1 - kernel, 0 - user
#
# Uses the following registers to pass to __exc:
# - r4 = sigcode
# - r6 = fault address
#-------------------------------------------------------------------------

__exc_access_write:
	mov.l	Lea_TEA, r1
	mov.l	@r1, r3
	mov.l	Lea_VM_USER_SPACE_BOUNDRY, r2
	cmp/hi	r2, r3
	bf		1f

	#
	# Acquire kernel and switch to bank0 with interrupts and FPU disabled
	#
	EXC_ENTERKERNEL
	stc		r3_bank, r6				! pick up fault address from above

	#
	# Fault address > VM_USER_SPACE_BOUNDRY - deliver signal
	#
	mov.l	Lea_SIGSEGV_code, r4
	bra		__exc
	nop

	#
	# Acquire kernel and switch to bank0 with interrupts and FPU disabled
	#
1:	EXC_ENTERKERNEL
	stc		r3_bank, r6				! pick up fault address from above

	#
	# Check for instruction address fault or alignment fault
	#
	mov.l	Lea_SIGBUS_code, r0

	#
	# Don't emulate if kernel/interrupt
	#
	tst		r14, r14
	bf		1f

	#
	# Don't emulate for instruction address fault (fault address == pc)
	#
	mov		#REG_PC, r1
	add		r11, r1
	mov.l	@r1, r2
	cmp/eq	r2, r6
	bt		1f

	#
	# Don't emulate if act->flags has _NTO_TF_ALIGN_FAULT set
	#
	mov.l	Lea_TFLAGS, r4
	mov.l	Lea_TF_ALIGN_FAULT, r5
	add		r12, r4
	mov.l	@r4, r4
	tst		r4, r5
	bf		1f

	#
	# Call emulate_alignment(act, SIGBUS_code) to handle misaligned access
	#
	mov		r0, r5
	mov.l	Lea_emulate_alignment, r0
	jsr		@r0
	mov		r11, r4

	tst		r0, r0
	bf		1f

	#
	# Fault was handled - pc has been adjusted to next instruction.
	#
	bra		__ker_exit
	nop

	#
	# Fault was not handled - deliver signal
	#
1:	bra		__exc
	mov		r0, r4

#-------------------------------------------------------------------------
# Common code for handling access/protection violation exceptions
#
# On entry:
# - executing in bank1 (SR.BL=1, SR.RB=1)
# - r3  = signal code corresponding to fault
# - r11 = saved context
# - r12 = actives[RUNCPU]
# - r14 = kernel/user mode indicator: 1 - kernel, 0 - user
#-------------------------------------------------------------------------

__exc_access:
	#
	# Get fault address
	#
	mov.l	Lea_SH_MMR_CCN_TEA, r1
	mov.l	@r1, r9

	#
	# Save ASID in r8
	#
	mov.l	Lea_SH_MMR_CCN_PTEH, r1
	mov.l	@r1, r0
	and		#0xff, r0
	mov		r0, r8

.ifdef	VARIANT_smp
	#
	# On SMP we can't acquire the kernel at this point.
	# We could have a TLB miss on one cpu that requires calling memmgr.fault
	# to resolve the fault (eg. a permanent mapping), but another cpu could
	# be in the kernel. If this happens we would spin waiting for the other
	# cpu to release the kernel, and may potentially deadlock.
	#
	# Instead, set cpupageptr[RUNCPU]->state to prevent preemption and then
	# acquire the kernel once we know what to do with the fault.
	#
	mov.l	Lea_SR_not_RB_BL, r1
	mov.l	Lea_SR_FD, r2
	stc		sr, r0
	and		r1, r0						! switch to bank0
	or		r2, r0						! disable FPU
	or		#SH_SR_IMASK, r0			! disable interrupts
	ldc		r0, sr
	mov.l	Lea_cpupageptr, r0
	SMP_GETCPU	r3
	SMP_ADD	r0, r3
	mov.l	@r0, r0						! cpupageptr[RUNCPU]
	mov		#1, r1
	mov.l	@(CPUPAGE_STATE, r0), r10	! remember previous state
	mov.l	r1, @(CPUPAGE_STATE, r0)
.else
	#
	# Acquire kernel and switch to bank0 with interrupts and FPU disabled
	#
	EXC_ENTERKERNEL
.endif
	stc		r3_bank, r6					! pick up signal code

	#
	# Enable interrupts and call memmgr.fault()
	#
	# FIXME_INTR: should this set to shadow_imask?
	mov.l	Lea_SR_not_IMASK, r1
	stc		sr, r0
	and		r1, r0
	ldc		r0, sr

	#
	# Set additional sigcode flags:
	# - SIGCODE_KERNEL if entered from kernel/interrupt
	# - SIGCODE_EXIT   if INKERNEL_EXIT was set
	# - SIGCODE_INXFER if xfer_handlers[] is non-NULL
	#
	tst		r14, r14
	bt		1f
	mov.l	Lea_SIGCODE_KERNEL, r0
	or		r0, r6

1:	mov		#INKERNEL_EXIT, r0
	tst		r13, r0
	bt		1f
	mov.l	Lea_SIGCODE_KEREXIT, r0
	or		r0, r6

1:	mov.l	Lea_xfer_handlers, r0
.ifdef	SMP_MSGOPT
	SMP_GETCPU	r3
	SMP_ADD	r0, r3						! &xfer_handlers[RUNCPU]
.endif
	mov.l	@r0, r0
	tst		r0, r0
	bt		1f
	mov.l	Lea_SIGCODE_INXFER, r0
	or		r0, r6

	#
	# Create fault_info on stack and call memmgr.fault(&info)
	#
1:	add		#-SIZEOF_FAULT_INFO, r15
	mov.l	Lea_aspaces_prp, r2
	mov.l	Lea_MEMMGR_FAULT, r1
	SMP_GETCPU	r3
	SMP_ADD	r2, r3
	mov.l	@r2, r2						! aspaces_prp[RUNCPU]
	mov.l	@r1, r0						! memmgr.fault
	mov.l	r2, @(FI_PRP, r15)			! info->prp
	mov.l	r9, @(FI_VADDR, r15)		! info->vaddr
	mov.l	r6, @(FI_SIGCODE, r15)		! info->sigcode
	mov.l	r8, @(FI_CPU_FLAGS, r15)	! info->cpu.flags
	jsr		@r0
	mov		r15, r4

	#
	# memmgr.fault() returns one of the following codes:
	# -1: fault was not handled - deliver the signal
	#  0: fault needs to be deferred - call PageWait()
	#  1: fault was handled - restore context
	#
	cmp/eq	#1, r0
	bf		__exc_access_fault

	#
	# Fault was handled - we can restore the faulting context
	#
	add		#SIZEOF_FAULT_INFO, r15
	stc		sr, r0
	or		#SH_SR_IMASK, r0
	ldc		r0, sr

.ifdef	VARIANT_smp
	#
	# Restore cpupageptr[RUNCPU]->state
	#
	mov.l	Lea_cpupageptr, r0
	SMP_GETCPU	r3
	SMP_ADD	r0, r3
	mov.l	@r0, r0						! cpupageptr[RUNCPU]
	mov.l	r10, @(CPUPAGE_STATE, r0)
.else
	#
	# Restore original inkernel
	#
	ldc		r13, r7_bank
.endif
	tst		r14, r14
	bt		0f
	bra		__ker_iret
	nop
0:	bra		__ker_exit
	nop

__exc_access_fault:
.ifdef	VARIANT_smp
	#
	# Acquire the kernel
	#
	mov		r0, r8
	mov.l	Lea_SR_RB_BL, r1
	stc		sr, r0
	or		r1, r0						! switch to bank1 for EXC_ENTERKERNEL
	or		#SH_SR_IMASK, r0
	ldc		r0, sr
	EXC_ENTERKERNEL

	#
	# Restore cpupageptr[RUNCPU]->state and re-enable interrupts
	#
	mov.l	Lea_cpupageptr, r0
	SMP_GETCPU	r3
	SMP_ADD	r0, r3
	mov.l	@r0, r0						! cpupageptr[RUNCPU]
	mov.l	r10, @(CPUPAGE_STATE, r0)
	mov.l	Lea_SR_not_IMASK, r1
	stc		sr, r0
	and		r1, r0
	ldc		r0, sr
	mov		r8, r0
.endif

	cmp/eq	#0, r0
	bt		__exc_access_pagewait
	#
	# Need to deliver signal - pick up sigcode to deliver
	#
	mov.l	Lea_SIGCODE_mask, r1
	mov.l	@(FI_SIGCODE, r15), r4		! info->sigcode
	bra		2f
	and		r1, r4

__exc_access_pagewait:
	#
	# Call PageFaultWait(&info) to block faulting thread and send a pulse
	# to the memmgr to process the fault.
	# Once the fault is resolved, the thread is unblocked via PageCont().
	#
	mov.l	Lea_PageFaultWait, r0
	jsr		@r0
	mov		r15, r4						! &info
	tst		r0, r0
	mov		r0, r4
	bt		2f
	#
	# PageFaultWait failed - pick up sigcode to deliver
	#
	mov.l	@(FI_SIGCODE, r15), r4		! info->sigcode
	mov.l	Lea_SIGCODE_mask, r1
	and		r1, r4

2:	add		#SIZEOF_FAULT_INFO, r15
	#
	# Disable interrupts and check if user/kernel fault
	#
	stc		sr, r0
	or		#SH_SR_IMASK, r0
	ldc		r0, sr
	tst		r14, r14
	bf		__exc_access_sys

	#
	# User fault.
	# If fault was handled, return via __ker_exit().
	# Otherwise, deliver the signal.
	#
	tst		r4, r4
	bf		1f
	bra		__ker_exit
	nop
1:	bra		__exc_usr
	mov		r9, r6

__exc_access_sys:
	#
	# Restore original inkernel value
	#
.ifdef	VARIANT_smp
	#
	# Restore only the inkernel bits - other cpus can change the intr count
	#
	mov.l	Lea_INKERNEL_bits, r2
	stc		r7_bank, r1
	and		r2, r13						! old inkernel bits
	not		r2, r2
0:	.word	0x0163						! movli.l @r1, r0
	and		r2, r0						! clear inkernel bits
	or		r13, r0						! set old inkernel bits
	.word	0x0173						! movco.l r0, @r1
	bf		0b
.else
	ldc		r13, r7_bank
.endif

	#
	# If kernel was locked, or we faulted in an interrupt handler we can
	# restore the faulting context if it was handled by memmgr.fault().
	# Otherwise, we have to crash since we cannot recover from the fault.
	#
.ifdef	VARIANT_smp
	mov		#INKERNEL_LOCK, r0
	tst		r0, r13
	bf		0f

	#
	# Must look at cpupageptr[RUNCPU]->state to check for interrupt
	#
	mov.l	Lea_cpupageptr, r0
	SMP_GETCPU	r3
	SMP_ADD	r0, r3						! &cpupageptr[RUNCPU]
	mov.l	@r0, r0
	mov.l	@(CPUPAGE_STATE, r0), r0	! cpupageptr[RUNCPU]->state
	tst		r0, r0
	bt		1f
.else
	mov		#INKERNEL_INTRMASK+INKERNEL_LOCK, r0
	tst		r0, r13
	bt		1f
.endif
0: 	tst		r4, r4
	bf		0f
	bra		__ker_iret
	nop	
0:	bra		__hardcrash
	nop

	#
	# Check if we were in specialret() processing
	#
1:	mov		#INKERNEL_SPECRET, r0
	tst		r0, r13
	bf		__exc_access_specret

	#
	# If the fault was handled, current thread is blocked in STATE_PAGEWAIT.
	# Call xfer restart handler if set, and restart the kernel call.
	#
	tst		r4, r4
	bf		1f
	bra		preempt
	nop
1:
	#
	# Fault was not handled.
	# Call xfer fault handler if set and fix up kernel call to return EFAULT.
	#
	bra		fixup_kcall
	nop

	#
	# Handle fault in specialret()
	#
__exc_access_specret:
	#
	# Fix up the kernel call if the fault was not handled
	#
	tst		r4, r4
	bt		0f
	bra		fixup_specret
	nop
0:

	#
	# Fault was handled and thread is blocked in STATE_PAGEWAIT
	# - clear INKERNEL_SPECRET
	# - call xfer restart handler if set
	# - return via __ker_exit to resume new active thread
	#
	mov		#INKERNEL_SPECRET, r2
	not		r2, r2
.ifdef	VARIANT_smp
	stc		r7_bank, r1
0:	.word	0x0163				! movli.l @r1, r0
	and		r2, r0				! clear INKERNEL_SPECRET
	.word	0x0173				! movco.l r0, @r1
	bf		0b
.else
	and		r2, r13				! clear INKERNEL_SPECRET
	ldc		r13, r7_bank
.endif

	#
	# Call restart handler if set
	#
	mov.l	Lea_xfer_handlers, r1
.ifdef	SMP_MSGOPT
	SMP_GETCPU	r3
	SMP_ADD	r1, r3				! &xfer_handlers[RUNCPU]
.endif
	mov.l	@r1, r0
	tst		r0, r0
	bt		1f
	mov		#0, r2
	mov.l	r2, @r1				! clear xfer_handlers
	mov.l	@(4, r0), r0		! get restart handler
	tst		r0, r0
	bt		1f
	mov		r12, r4
	jsr		@r0					! call handler(act, context)
	mov		r11, r5

1:	bra		__ker_exit
	nop

	.align 2

Lea_SIGSEGV_code:			.long	SIGSEGV+ (SEGV_ACCERR*256) + (FLTBOUNDS*65536)
Lea_SIGBUS_code:			.long	SIGBUS+ (BUS_ADRALN*256) + (FLTACCESS*65536)
Lea_TEA:					.long	SH_MMR_CCN_TEA
Lea_VM_USER_SPACE_BOUNDRY:	.long	VM_USER_SPACE_BOUNDRY
Lea_SR_not_IMASK:			.long	~SH_SR_IMASK
Lea_emulate_alignment:		.long	emulate_alignment
Lea_TFLAGS:					.long	TFLAGS
Lea_TF_ALIGN_FAULT:			.long	_NTO_TF_ALIGN_FAULT
Lea_SH_MMR_CCN_PTEH:		.long	SH_MMR_CCN_PTEH
Lea_SH_MMR_CCN_TEA:			.long	SH_MMR_CCN_TEA
Lea_SIGCODE_KERNEL:			.long	SIGCODE_KERNEL
Lea_SIGCODE_KEREXIT:		.long	SIGCODE_KEREXIT
Lea_SIGCODE_INXFER:			.long	SIGCODE_INXFER
Lea_SIGCODE_mask:			.long	~(SIGCODE_KERNEL|SIGCODE_KEREXIT|SIGCODE_INXFER)
Lea_MEMMGR_FAULT:			.long	MEMMGR_FAULT+memmgr
Lea_PageFaultWait:			.long	PageFaultWait
Lea_aspaces_prp:			.long	aspaces_prp
Lea_xfer_handlers:			.long	xfer_handlers
Lea_intrevent_pending:		.long	intrevent_pending
Lea_REG_OFF_PC:				.long	REG_OFF+REG_PC
.ifdef	VARIANT_smp
Lea_cpupageptr:				.long	cpupageptr
Lea_SR_not_RB_BL:			.long	~(SH_SR_RB | SH_SR_BL)
Lea_SR_RB_BL:				.long	SH_SR_RB | SH_SR_BL
Lea_SR_FD:					.long	SH_SR_FD
Lea_INKERNEL_bits:			.long	(INKERNEL_NOW|INKERNEL_LOCK|INKERNEL_SPECRET|INKERNEL_EXIT)
.endif

#-------------------------------------------------------------------------
# EXPEVT 180 - general illegal instruction exception
# EXPEVT 1A0 - slot illegal instruction exception
#
# On entry:
# - executing in bank 1 (SR.BL=1, SR.RB=1):
# - r11 = saved context
# - r12 = actives[RUNCPU]
# - r14 = kernel/user mode indicator: 1 - kernel, 0 - user
#-------------------------------------------------------------------------
__exc_ill_instr:
__exc_ill_instr_slot:
	#
	# Acquire kernel and switch to bank0 with interrupts and FPU disabled
	#
	EXC_ENTERKERNEL

	#
	# Check for special cases
	#
	mov		#REG_PC, r0
	mov.l	@(r0, r11), r1				! saved pc
	mov.w	@r1, r0						! faulting instruction
	cmp/eq	#-3, r0						! opcode == 0xfffd
	bf		__exc_ill_emulate

	#
	# __inline_DebugKDBreak() opcode - deliver SIGTRAP
	#
	# FIXME: check this is the right thing to do...
	#
	mov.l	Lei_SIGTRAP_code, r4
	bra 	__exc
	nop

__exc_ill_emulate:
	#
	# Call emulate_instruction() to see if instruction can be emulated.
	#
	mov.l	Lei_emulate_instruction, r0
	mov.l	Lei_SIGILL_code, r5
	jsr		@r0
	mov		r12, r4

	#
	# If emulate_instruction() returns 0, it will have advanced the PC to
	# the next instruction, so return via __ker_exit()
	#
	tst		r0, r0
	bf		0f
	bra		__ker_exit
	nop

	#
	# Instruction can't be emulated - deliver SIGILL
	#
0:	mov.l	Lei_SIGILL_code, r4
	bra 	__exc
	nop

#-------------------------------------------------------------------------
#
#-------------------------------------------------------------------------
__exc_usr_break_enter:
	#
	# Save context, acquire kernel and switch to bank0 with interrupts disabled
	#
	EXC_SAVECONTEXT
#user break
# user break point should not be put on a delayed slot!!!
__exc_usr_break:
	#
	# Acquire kernel and switch to bank0 with interrupts and FPU disabled
	#
	EXC_ENTERKERNEL

	# reduce the ip by KER_ENTRY_SIZE (because it is implemented by a trap#)
	mov		r11, r1
	add		#REG_PC, r1
	mov.l	@r1, r0
	add		#-KER_ENTRY_SIZE, r0
	mov.l	r0, @r1

# optimize it later NIY
	mov.l	Lei_SR_FD, r1
	stc		sr, r0
	or		r1, r0
	ldc		r0, sr

	mov.l	Lei_SIGTRAP_code, r4
	bra 	__exc
	nop

	.align 2

Lei_SR_not_RB_BL_FD:		.long	~(SH_SR_RB|SH_SR_BL|SH_SR_FD)
Lei_SIGTRAP_code:			.long	SIGTRAP + (TRAP_BRKPT*256) + (FLTBPT*65536)
Lei_SIGILL_code:			.long	SIGILL + (ILL_PRVOPC*256) + (FLTPRIV*65536)
Lei_emulate_instruction:	.long	emulate_instruction
Lei_SR_FD:					.long	SH_SR_FD

#-------------------------------------------------------------------------
# Unexpected exception handler
#
# On entry:
# - executing in bank 1 (SR.BL=1, SR.RB=1):
# - r11 = saved context
# - r12 = actives[RUNCPU]
# - r14 = kernel/user mode indicator: 1 - kernel, 0 - user
#-------------------------------------------------------------------------

__exc_unexpected:
	#
	# Acquire kernel and switch to bank0 with interrupts and FPU disabled
	#
	EXC_ENTERKERNEL

	#
	# Put exception code in upper 16 bits of signal code
	#
	mov.l	Leu_MMR_CCN_EXPEVT, r1
	mov.l	@r1, r4
	shll16	r4

	#
	# Fall into __hardcrash
	#

#-------------------------------------------------------------------------
# Crash the system:
# - invoke kernel debugger (if present)
# - call shutdown() if debugger is not present or didn't handle the fault
#
# r4 contains the signal code: signo | (si_code*256) | (fault_num*65536)
#
#-------------------------------------------------------------------------

__hardcrash:
.ifdef	VARIANT_smp
	# FIXME: is this correct
!	stc		r7_bank, r0
!	mov.l	r13, @r0
.else
	ldc		r13, r7_bank
.endif
	mov.l	Leu_SIGCODE_FATAL, r0
	or		r0, r4

	#
	# Call kdebug_callout(sigcode, context)
	#
	mov.l	Leu_kdebug_callout, r0
	jsr		@r0
	mov		r11, r5
	cmp/eq	#0, r0
	bf		1f
	bra		__ker_iret
	nop
1:
	#
	# Debugger not present or it didn't handle the fault.
	#
	# Call shutdown(context) to halt system with shutdown message.
	#
	mov.l	Leu_shutdown, r0
	jsr		@r0
	mov		r11, r5
1:	bra		1b
	nop

	.align 2

Leu_MMR_CCN_EXPEVT:	.long	SH_MMR_CCN_EXPEVT
Leu_SIGCODE_FATAL:	.long	SIGCODE_FATAL
Leu_shutdown:		.long	shutdown
Leu_kdebug_callout:	.long	kdebug_callout

#-------------------------------------------------------------------------
# Common exception handler.
#
# On entry:
# - interrupts are disabled
# - kernel acquired INKERNEL_NOW | INKERNEL_LOCK
# - r4 contains signal code
# - r6 contains address associated with fault
# - r11 = saved context
# - r12 = actives[RUNCPU]
# - r13 = inkernel on exception entry
# - r14 = kernel/user mode indicator: 1 - kernel, 0 - user
#-------------------------------------------------------------------------

__exc:
	tst		r14, r14
	bf		__exc_sys

__exc_usr:
	#
	# Enable interrupts
	#
	mov.l	Lex_SR_not_IMASK, r1
	stc		sr, r0
	and		r1, r0
	ldc		r0, sr

	#
	# Call usr_fault(signo, act, addr) to deliver the signal
	#
	mov.l	Lex_usr_fault, r0
	jsr		@r0
	mov		r12, r5

	mov.l	Lex_ker_exit, r0
	jmp		@r0
	nop

__exc_sys:
.ifdef	VARIANT_smp
	#
	# On SMP we have to check cpupageptr[RUNCPU]->state
	#
	mov.l	Lex_cpupageptr, r1
	SMP_GETCPU	r3
	SMP_ADD	r1, r3
	mov.l	@r1, r0						! cpupageptr[RUNCPU]
	mov.l	@(CPUPAGE_STATE, r0), r1
	tst		r1, r1
!	bf		fixup_intr
bf __hardcrash
	mov		r13, r0
.else
	mov		r13, r0
	tst		#INKERNEL_INTRMASK, r0
	bf		fixup_intr
.endif

	tst		#INKERNEL_NOW, r0
	bt		__hardcrash

	tst		#INKERNEL_LOCK, r0
	bf		__hardcrash

	tst		#INKERNEL_SPECRET, r0
	bf		fixup_specret

	#
	# Give the kernel debugger a chance to deal with the exception
	#
	# If kdebug_opt returns, it means the debugger wasn't present or it
	# didn't handle the fault, so make the current kernel call return EFAULT
	#
	# FIXME: not sure this is always correct, but that's what some of the
	#        other processors do. Perhaps we should set SIGCODE_FATAL?
	#        This would mean:
	#        - if debugger handles it, we resume the faulting context
	#        - if debugger doesn't handle it, we shutdown
	#
	mov.l	Lex_SIGCODE_KERNEL, r0
	bsr		kdebug_opt
	or		r0, r4
	bra		fixup_kcall
	nop

#-------------------------------------------------------------------------
# Fix up fault in specialret():
# - turn off currently active flag
# - fix up kernel call to return EFAULT
#
#	lock_kernel();
#	bitclr_inkernel(INKERNEL_SPECRET);
#	act->flags &= ~inspecret;
#	fixup_kcall();
#-------------------------------------------------------------------------

fixup_specret:
.ifdef	VARIANT_smp
	stc		r7_bank, r1
	mov.l	Lex_not_INKERNEL_SPECRET, r2
0:	.word	0x0163							! movli.l @r1, r0
 	or		#INKERNEL_LOCK, r0
	and		r2, r0
	.word	0x0173							! movco.l r0, @r1
	bf		0b
.else
	stc		r7_bank, r0
 	or		#INKERNEL_LOCK, r0
	and		#~INKERNEL_SPECRET, r0
	ldc		r0, r7_bank
.endif

	#
	# Turn off the flag currently being processed in specialret()
	#
	mov.l	Lex_inspectret, r1
	mov.l	Lex_TFLAGS, r0
	mov.l	@r1, r2
	mov.l	@(r0, r12), r3
	not		r2, r2
	and		r2, r3
	mov.l	r3, @(r0, r12)

	#
	# Fall through to fixup_kcall
	#

#-------------------------------------------------------------------------
# Fix up fault in kernel call:
# - call xfer fault handler if set
# - set syscall return to EFAULT
# - advance pc to error handling code in syscall stub if necessary
#
#	lock_kernel();
#	if (xfer = xfer_handlers[]) {
#		xfer_handlers[] = 0;
#		xfer->fault(act, context, sigcode);
#	}
#	act->timeout_flags = 0;
#	act->reg.gpr[R0] = EFAULT;
#	if ((act->flags & _NTO_TF_KERERR_SET) == 0) {
#		act->flags |= _NTO_TF_KERERR_SET;
#		act->reg.gpr[PC] += KERERR_SKIPAHEAD;
#	}
#   __ker_exit();
#
# On entry:
# - r4  = sigcode
# - r11 = context
# - r12 = act
#-------------------------------------------------------------------------

fixup_kcall:
.ifdef	VARIANT_smp
	stc		r7_bank, r1
0:	.word 	0x0163						! movli.l @r1, r0
	or		#INKERNEL_LOCK, r0
	.word	0x0173						! movco.l r0, @r1
	bf		0b
.else
	stc		r7_bank, r0
	or		#INKERNEL_LOCK, r0
	ldc		r0, r7_bank
.endif

	mov.l	Lex_xfer_handlers, r1
.ifdef	SMP_MSGOPT
	SMP_GETCPU	r3
	SMP_ADD	r1, r3						! &xfer_handlers[RUNCPU]
.endif
	mov.l	@r1, r0
	tst		r0, r0
	bt		1f
	mov		#0, r2
	mov.l	r2, @r1						! clear xfer_handlers[RUNCPU]
	mov.l	@r0, r0						! get fault handler
	tst		r0, r0
	bt		1f
	mov		r4, r6
	mov		r11, r5
	jsr		@r0							! handler(act, context, sigcode)
	mov		r12, r4

1:	sub		r2, r2
	mov.l	Lex_TIMEOUT_FLAGS, r0
	mov.l	r2, @(r0, r12)				! act->timeout_flags = 0
	mov.l	Lex_REG_OFF_R0, r0
	mov		#ERRNO_EFAULT, r2
	mov.l	r2, @(r0, r12)				! act->reg.gpr[R0] = EFAULT

	mov.l	Lex_TFLAGS, r0
	mov.l	@(r0, r12), r2
	mov.l	Lex_TF_KERERR_SET, r1
	tst		r1, r2
	bf		1f
	or		r1, r2
	mov.l	r2, @(r0, r12)				! act->flags |= _NTO_TF_KERERR_SET
	mov.l	Lex_REG_OFF_PC, r0
	mov.l	@(r0, r12), r2
	add		#KERERR_SKIPAHEAD, r2
	mov.l	r2, @(r0, r12)				! act->reg.gpr[PC] += KERERR_SKIPAHEAD

1:	mov.l	Lex_ker_exit, r0
	jmp		@r0
	nop

#-------------------------------------------------------------------------
# Fixup fault in interrupt handler:
# - if kernel debugger claims to have handled the fault, restore context
# - otherwise crash
#-------------------------------------------------------------------------
fixup_intr:
	mov.l	Lex_SIGCODE_INTR, r0
	bsr		kdebug_opt
	or		r0, r4
	cmp/eq	#0, r0
	bf		1f

	#
	# Debugger claims to have recovered from the fault
	#
	mov.l	Lex_ker_iret, r0
	jmp		@r0
	nop

	#
	# Can't recover from interrupt handler crash
	#
1:	bra	__hardcrash
	nop

	.align 2

Lex_usr_fault:				.long	usr_fault
Lex_SR_not_IMASK:			.long	~SH_SR_IMASK
Lex_xfer_handlers:			.long	xfer_handlers
Lex_SIGCODE_KERNEL:			.long	SIGCODE_KERNEL
Lex_TIMEOUT_FLAGS:			.long	TIMEOUT_FLAGS
Lex_TFLAGS:					.long	TFLAGS
Lex_TF_KERERR_SET:			.long	_NTO_TF_KERERR_SET
Lex_REG_OFF_PC:				.long	REG_OFF+REG_PC
Lex_REG_OFF_R0:				.long	REG_OFF+REG_GR
Lex_inspectret:				.long	inspecret
Lex_SIGCODE_INTR:			.long	SIGCODE_INTR
Lex_ker_exit:				.long	__ker_exit
Lex_ker_iret:				.long	__ker_iret
.ifdef	VARIANT_smp
Lex_not_INKERNEL_SPECRET:	.long	~INKERNEL_SPECRET
Lex_cpupageptr:				.long	cpupageptr
.endif


#-------------------------------------------------------------------------
# FIXME: figure out exactly what is going on here...
#-------------------------------------------------------------------------
__handle_kdbmsg:
	#
	# Save context, acquire kernel and switch to bank0 with interrupts disabled
	#
	EXC_SAVECONTEXT
	EXC_ENTERKERNEL

	mov.l	Lkd_kebug_opt, r1
	mov.l	Lkd_SIGCODE_KERNEL, r0
	jsr		@r1
	or		r0, r4

	# No debugger, keep on going
	mov.l	Lkd_ker_iret, r0
	jmp		@r0
	nop

#-------------------------------------------------------------------------
# Invoke the kernel debugger (if present)
#-------------------------------------------------------------------------
kdebug_opt:
	sts		pr, r8
.ifdef	VARIANT_smp
	# FIXME: is this correct
	stc		r7_bank, r0
	mov.l	r13, @r0
.else
	ldc		r13, r7_bank	! restore original inkernel value
.endif

	mov.l	Lkd_kdebug_callout, r0
	jsr		@r0
	mov		r11, r5

	#
	# If the debugger indicates it handled the fault, restore the context
	#
	tst		r0, r0
	bf		1f
	mov.l	Lkd_ker_iret, r0
	jmp		@r0
	nop
1:
	lds		r8,pr

	#
	# Turn off special flag bits in signal code
	#
	mov.l	Lkd_not_SIGCODE_FLAGS_MASK, r1
	rts
	and		r1, r0
	.align 2
Lkd_kdebug_callout:				.long	kdebug_callout
Lkd_not_SIGCODE_FLAGS_MASK:		.long	0x00ffffff
Lkd_SR_not_RB_BL_FD:			.long	~(SH_SR_RB|SH_SR_BL|SH_SR_FD)
Lkd_SIGCODE_KERNEL:				.long	SIGCODE_KERNEL
Lkd_kebug_opt:					.long	kdebug_opt
Lkd_ker_iret:					.long	__ker_iret


#-------------------------------------------------------------------------
# cpu_force_fpu_save: force a flush of current FPU context
#
# r4 = THREAD pointer
#-------------------------------------------------------------------------

routine_start cpu_force_fpu_save, 1
	#
	# Make sure FPU is enabled and interrupts are disabled
	#
	mov.l	Lfs_SR_not_FD, r1
	stc		sr, r0
	mov		r0, r6						! save previous value
	and		r1, r0
	or		#SH_SR_IMASK, r0
	ldc		r0, sr

.ifdef	VARIANT_smp
	mov.l	Lfs_FPUDATA, r0
	add		r0, r4						! &thp->fpudata
	mov		r4, r5						! remember address for code below
	mov.l	@r4, r4
	#
	# Mask the BUSY/CPU bits in fpudata
	#
	mov.l	Lfs_FPUDATA_MASK, r1
	and		r1, r4
.else
	mov.l	Lfs_FPUDATA, r0
	mov.l	@(r0, r4), r4
.endif

	SAVE_FPU_REGISTERS	r4

.ifdef	VARIANT_smp
	#
	# Clear BUSY/CPU bits in fpudata to indicate context is no longer active
	# r4 is now the fpudata value without the BUSY/CPU bits
	#
	mov.l	r4, @r5					! r5 = &thp->fpudata set above
.endif

	#
	# Restore SR FPU disable state
	#
	ldc		r6, sr
	rts
	nop

	.align 2

Lfs_SR_not_FD:			.long	~SH_SR_FD
Lfs_FPUDATA:			.long	FPUDATA
.ifdef	VARIANT_smp
Lfs_FPUDATA_MASK:		.long	FPUDATA_MASK
.endif

routine_end cpu_force_fpu_save


#
# restore_fpu_registers(SH_FPU_REGISTERS* p)
#
# restore the fpu registers from the context pointed to by the parameter.
#
routine_start restore_fpu_registers, 1

	RESTORE_FPU_REGISTERS	r4
	rts
	nop

routine_end restore_fpu_registers

